Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_pulsar_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-hv4db-1g0bg --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d2b597793d6e825943469ba40aa3ae128139e5e8" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-hv4db" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=93a444ed-f8c0-46f2-999c-eb721888a099 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-hv4db GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-100f7bb7add7cfb003dff5768bf20348b91a6b6cef38154d1625c661ef800a11 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-hv4db GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build466374606=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-hv4d-1m8lr bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 35496 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: a83ccb8ada12a531 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 22 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Sat May 04 02:55:57 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/model/kv.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sourcemanager/manager.go, cdc/redo/reader/reader.go, cdc/sink/dmlsink/factory/factory.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/kafka-consumer/main.go, cmd/pulsar-consumer/main.go, cmd/storage-consumer/main.go, errors.toml, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/errors/cdc_errors.go, pkg/errors/helper.go, tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml, tests/integration_tests/changefeed_dup_error_restart/conf/workload, tests/integration_tests/changefeed_dup_error_restart/run.sh, tests/integration_tests/force_replicate_table/run.sh, tests/integration_tests/open_protocol_handle_key_only/data/data.sql, tests/integration_tests/run_group.sh] [Pipeline] echo diff file not matched: cdc/model/kv.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534) 203635712 bytes in 1.24 secs (163635108 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/.git/ .git HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) POST git-upload-pack (656 bytes) POST git-upload-pack (973 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master * [new ref] refs/pull/10919/head -> origin/pr/10919/head HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) ๐Ÿšง Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) ๐Ÿšง Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..b297eae37 Fast-forward cdc/model/kv.go | 5 + cdc/model/sink.go | 35 ++- cdc/model/sink_test.go | 9 +- cdc/processor/processor.go | 21 +- cdc/processor/sinkmanager/manager.go | 5 + cdc/processor/sourcemanager/manager.go | 66 +++- cdc/redo/reader/reader.go | 21 +- cdc/sink/dmlsink/factory/factory.go | 8 +- cdc/sink/dmlsink/txn/mysql/mysql.go | 89 +++--- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 2 +- cmd/kafka-consumer/main.go | 4 +- cmd/pulsar-consumer/main.go | 17 +- cmd/storage-consumer/main.go | 4 +- errors.toml | 5 + pkg/applier/redo.go | 303 +++++++++++++++++- pkg/applier/redo_test.go | 347 ++++++++++++++++++++- pkg/errors/cdc_errors.go | 4 + pkg/errors/helper.go | 19 ++ .../conf/diff_config.toml | 29 ++ .../changefeed_dup_error_restart/conf/workload | 13 + .../changefeed_dup_error_restart/run.sh | 54 ++++ .../integration_tests/force_replicate_table/run.sh | 4 +- .../open_protocol_handle_key_only/data/data.sql | 46 +-- tests/integration_tests/run_group.sh | 5 +- 24 files changed, 975 insertions(+), 140 deletions(-) create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/workload create mode 100755 tests/integration_tests/changefeed_dup_error_restart/run.sh ๐Ÿงพ Pre-merged result: b297eae37cdb06fa18bc5388b04d2564a81a0b87 b297eae37 check dd5a6720f fix 4085e5de5 check test โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache saved successfully (git/pingcap/tiflow/rev-be15534-b297eae) 203837440 bytes in 9.40 secs (21683271 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 461 0 --:--:-- --:--:-- --:--:-- 465 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 912 0 --:--:-- --:--:-- --:--:-- 931 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1478 0 --:--:-- --:--:-- --:--:-- 1518 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 442 0 --:--:-- --:--:-- --:--:-- 445 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz 2024-05-04 10:56:37 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz [536570515/536570515] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-04 10:56:55 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz 2024-05-04 10:57:36 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz [919098782/919098782] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz 2024-05-04 10:58:04 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz [456057803/456057803] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-04 10:58:10 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-04 10:58:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-04 10:58:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-04 10:58:14 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-04 10:58:18 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-04 10:58:34 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 10:58 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 10:58 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 10:58 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 10:58 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 10:58 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 10:58 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 10:58 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 10:58 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 10:58 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 10:58 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 02:58:37" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/google/uuid v1.6.0 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/google/btree v1.1.2 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/robfig/cron v1.2.0 go: downloading golang.org/x/sys v0.19.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading golang.org/x/text v0.14.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_pulsar_consumer ']' + make pulsar_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:00:35" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:00:42" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:00:42" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:00:42" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:00:42" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading go.uber.org/dig v1.13.0 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:00:42" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-54-gb297eae37" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 03:00:42" -X "github.com/pingcap/tiflow/pkg/version.GitHash=b297eae37cdb06fa18bc5388b04d2564a81a0b87" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-54-gb297eae37"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 11:05 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 10:58 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 4 11:05 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 11:05 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 11:00 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 11:01 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 11:00 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 4 11:01 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-54-gb297eae37 Git Commit Hash: b297eae37cdb06fa18bc5388b04d2564a81a0b87 Git Branch: HEAD UTC Build Time: 2024-05-04 03:00:42 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-pulsar-test/rev-be15534-b297eae) 1194907648 bytes in 22.62 secs (52835548 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 11:05 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 10:58 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 11:05 bin -rwxr-xr-x. 1 jenkins jenkins 220M May 4 11:05 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 11:05 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 11:00 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 11:01 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 11:00 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 11:05 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 4 11:05 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 11:05 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 11:05 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 11:05 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 4 11:05 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 4 11:05 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 4 11:05 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 4 11:05 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 4 11:05 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 4 11:05 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 4 11:05 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 4 11:05 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 4 11:05 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 4 11:01 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 4 11:05 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 4 11:05 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 4 11:05 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 4 11:05 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 4 11:05 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 4 11:05 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M May 4 11:05 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 4 11:05 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 11:05 share -rwxr-xr-x. 1 jenkins jenkins 32M May 4 11:05 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M May 4 11:05 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 4 11:05 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 4 11:05 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 4 11:05 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 4 11:05 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 67.46 secs (55298039 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-zmhz-2zfc4 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-0hqq-5dxcz Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-zmhz-2zfc4 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-zmhz3-f335v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "470b24bbaa34534503c9e6f7b400777b8317f534" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-zmhz3" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-zmhz-2zfc4" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-zmhz-2zfc4" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-zmhz-2zfc4" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-zmhz-2zfc4 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35 [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1789209a; decorates RemoteLauncher[hudson.remoting.Channel@66c4d20c:JNLP4-connect connection from 10.233.70.227/10.233.70.227:42160] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-b63nw-mgd5n --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "013cf00264e3b70dfe60633121d73619379cf186" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-b63nw" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] checkout Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git The recommended git tool is: git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4adf4848; decorates RemoteLauncher[hudson.remoting.Channel@6e6418a1:JNLP4-connect connection from 10.233.106.143/10.233.106.143:53128] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-f97zs-mv2wq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "157a63d2b0206cf910eeaa27c89ed01f5990afff" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-f97zs" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] timeout Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Timeout set to expire in 40 min [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow The recommended git tool is: git [Pipeline] { [Pipeline] cache Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-0hqq-5dxcz is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-0hqqd-jhhpd --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "78d8813834c3c61b855562577544c0dee235a75b" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-0hqqd" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-0hqq-5dxcz" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-0hqq-5dxcz" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-0hqq-5dxcz" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-0hqq-5dxcz in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@47b4668d; decorates RemoteLauncher[hudson.remoting.Channel@7ff23d09:JNLP4-connect connection from 10.233.108.35/10.233.108.35:57102] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-sbb7p-d973m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c8f16214990f3928866209799acc78fcac67eea6" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-sbb7p" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-blcq2-cgbrc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1457c09e23cb3aaf4c24e02c87bf66ccaea0cce5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-blcq2" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-7r7sz-1hjc8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "37c6ddffea9ef600b52742d11f4de6ea3cbb4f3a" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-7r7sz" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-j8s10-vdp4k --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3974732487d38099eec3b04304b9f8d9ba0b523d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-j8s10" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-nrk1j-vvh11 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "79fed43af5923ff3ba1fb6c42fb5e5bfa0aaae51" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-nrk1j" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 22.42 secs (166408793 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] { [Pipeline] withEnv + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G01 Run cases: open_protocol_handle_key_only PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=4feced26-c157-40dd-bfc3-04d74b75ee33 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-zmhz-2zfc4 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-zmhz3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-zmhz3 pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-zmhz-2zfc4 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-zmhz-2zfc4 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:07:34 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] { The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout [Pipeline] stage The recommended git tool is: git [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1b61ff0c; decorates RemoteLauncher[hudson.remoting.Channel@1ae05828:JNLP4-connect connection from 10.233.100.9/10.233.100.9:53050] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7e9863b0; decorates RemoteLauncher[hudson.remoting.Channel@2ff49ce6:JNLP4-connect connection from 10.233.93.208/10.233.93.208:53640] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@24374563; decorates RemoteLauncher[hudson.remoting.Channel@2edd8b9d:JNLP4-connect connection from 10.233.86.99/10.233.86.99:59066] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4e28cd89; decorates RemoteLauncher[hudson.remoting.Channel@5df49fb7:JNLP4-connect connection from 10.233.107.98/10.233.107.98:59216] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@20d5e406; decorates RemoteLauncher[hudson.remoting.Channel@859783b:JNLP4-connect connection from 10.233.88.183/10.233.88.183:59686] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2ac0843d; decorates RemoteLauncher[hudson.remoting.Channel@7c00e3e8:JNLP4-connect connection from 10.233.67.92/10.233.67.92:60672] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 \033[0;36m<<< Run all test success >>>\033[0m Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-xns4r-ccw61 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "8e53132bfbdc4f32d671ac8e1d2bab17cdcac603" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-xns4r" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-t0rdr-52mz5 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "556448cafe4df962c7448d03416090bd4aff7d41" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-t0rdr" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-h6bt9-cvdsl --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "93a82bc104d3eb03fda0b805fea8be76628288a0" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-h6bt9" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-79hx9-whg77 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "96c0688ac0f31d311a58a59ca0341a0896290153" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-79hx9" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-cnnh9-4vhv0 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1cf34c4eb25290fb8c52335a4f07842b094a520f" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-cnnh9" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-dhfzd-g1s1v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b22efb66ce0f6d69ec6b1804a70d04991f8e6878" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-dhfzd" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-rl3x1-ndrzj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "5ba16be707e3efbb58400e393282d9a11db7572e" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-rl3x1" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-4f64-16spm Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-ptr6b-qm55r --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d95eb6f187b9ffa94945c968886c2e21b439e264" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-ptr6b" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-4f64-16spm is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-4f64p-dzvvs --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "35f4564cbd7e49e389555360aabc3423772cd2c5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-4f64p" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-4f64-16spm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-4f64-16spm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-4f64-16spm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-4f64-16spm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 20.31 secs (183646297 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 17.76 secs (210070231 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=d0eabc4e-85de-47f1-aa99-ad49cc27e9b4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-b63nw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-b63nw pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=dc1ddc76-5d4c-4dfd-af6f-97aa4cb27be8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-f97zs GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7 pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-f97zs GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // cache [Pipeline] } [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] // dir [Pipeline] } [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@306b1739; decorates RemoteLauncher[hudson.remoting.Channel@42493a80:JNLP4-connect connection from 10.233.107.67/10.233.107.67:48624] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@264a2892; decorates RemoteLauncher[hudson.remoting.Channel@12d5a588:JNLP4-connect connection from 10.233.106.121/10.233.106.121:54560] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3c83e0f; decorates RemoteLauncher[hudson.remoting.Channel@b6d8b6a:JNLP4-connect connection from 10.233.70.163/10.233.70.163:56674] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@35f9bb78; decorates RemoteLauncher[hudson.remoting.Channel@29f25ed7:JNLP4-connect connection from 10.233.67.166/10.233.67.166:38218] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@65a8ff3c; decorates RemoteLauncher[hudson.remoting.Channel@6c6057a8:JNLP4-connect connection from 10.233.93.107/10.233.93.107:49036] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@75a11df4; decorates RemoteLauncher[hudson.remoting.Channel@108521d2:JNLP4-connect connection from 10.233.108.133/10.233.108.133:58422] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@45acc0b6; decorates RemoteLauncher[hudson.remoting.Channel@2eadb79a:JNLP4-connect connection from 10.233.100.254/10.233.100.254:45828] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@38ffb163; decorates RemoteLauncher[hudson.remoting.Channel@77706e25:JNLP4-connect connection from 10.233.73.200/10.233.73.200:36878] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@414276ae; decorates RemoteLauncher[hudson.remoting.Channel@6b5a717:JNLP4-connect connection from 10.233.127.26/10.233.127.26:51870] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d064ca4940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:1358, start at 2024-05-04 11:08:34.498453822 +0800 CST m=+5.088173260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:10:34.504 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:08:34.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:58:34.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d064ca4940015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:1358, start at 2024-05-04 11:08:34.498453822 +0800 CST m=+5.088173260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:10:34.504 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:08:34.469 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:58:34.469 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d064ca798000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:1441, start at 2024-05-04 11:08:34.678458173 +0800 CST m=+5.215552784 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:10:34.686 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:08:34.662 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:58:34.662 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d064ca6d80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:1303, start at 2024-05-04 11:08:34.631251182 +0800 CST m=+5.357449334 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:10:34.639 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:08:34.614 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:58:34.614 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d064ca6d80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:1303, start at 2024-05-04 11:08:34.631251182 +0800 CST m=+5.357449334 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:10:34.639 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:08:34.614 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:58:34.614 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d064ca7940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:1382, start at 2024-05-04 11:08:34.687571048 +0800 CST m=+5.359576578 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:10:34.695 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:08:34.661 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:58:34.661 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2829.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2833.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449522464985972737 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522464985972737 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:08:39 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.28662868.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 15.72 secs (237336693 bytes/sec) [Pipeline] { [Pipeline] cache + set +x + tso='449522465496104961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522465496104961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:08:41 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.28702872.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:08:42 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffe5dfaf-a526-4d81-a0d6-563df73e95d8 {"id":"ffe5dfaf-a526-4d81-a0d6-563df73e95d8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792119} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4192ff85d0 ffe5dfaf-a526-4d81-a0d6-563df73e95d8 /tidb/cdc/default/default/upstream/7364976018398550598 {"id":7364976018398550598,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffe5dfaf-a526-4d81-a0d6-563df73e95d8 {"id":"ffe5dfaf-a526-4d81-a0d6-563df73e95d8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792119} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4192ff85d0 ffe5dfaf-a526-4d81-a0d6-563df73e95d8 /tidb/cdc/default/default/upstream/7364976018398550598 {"id":7364976018398550598,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffe5dfaf-a526-4d81-a0d6-563df73e95d8 {"id":"ffe5dfaf-a526-4d81-a0d6-563df73e95d8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792119} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4192ff85d0 ffe5dfaf-a526-4d81-a0d6-563df73e95d8 /tidb/cdc/default/default/upstream/7364976018398550598 {"id":7364976018398550598,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:08:42 CST 2024] <<<<<< START pulsar cluster in foreign_key case >>>>>>' [Sat May 4 11:08:42 CST 2024] <<<<<< START pulsar cluster in foreign_key case >>>>>> + workdir=/tmp/tidb_cdc_test/foreign_key + cluster_type=normal + cd /tmp/tidb_cdc_test/foreign_key + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/foreign_key/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/foreign_key/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/foreign_key/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/foreign_key/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/foreign_key/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/foreign_key/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/foreign_key/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/foreign_key/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/foreign_key/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:08:42 CST 2024] <<<<<< START pulsar cluster in normal mode in foreign_key case >>>>>>' [Sat May 4 11:08:42 CST 2024] <<<<<< START pulsar cluster in normal mode in foreign_key case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/foreign_key/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/foreign_key/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/foreign_key/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:08:44 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb1fe7a1-a8e6-4aaf-85d4-c08471fcdd63 {"id":"eb1fe7a1-a8e6-4aaf-85d4-c08471fcdd63","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792121} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4192fe22d7 eb1fe7a1-a8e6-4aaf-85d4-c08471fcdd63 /tidb/cdc/default/default/upstream/7364976014090214956 {"id":7364976014090214956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb1fe7a1-a8e6-4aaf-85d4-c08471fcdd63 {"id":"eb1fe7a1-a8e6-4aaf-85d4-c08471fcdd63","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792121} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4192fe22d7 eb1fe7a1-a8e6-4aaf-85d4-c08471fcdd63 /tidb/cdc/default/default/upstream/7364976014090214956 {"id":7364976014090214956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eb1fe7a1-a8e6-4aaf-85d4-c08471fcdd63 {"id":"eb1fe7a1-a8e6-4aaf-85d4-c08471fcdd63","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792121} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4192fe22d7 eb1fe7a1-a8e6-4aaf-85d4-c08471fcdd63 /tidb/cdc/default/default/upstream/7364976014090214956 {"id":7364976014090214956,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:08:44 CST 2024] <<<<<< START pulsar cluster in row_format case >>>>>>' [Sat May 4 11:08:44 CST 2024] <<<<<< START pulsar cluster in row_format case >>>>>> + workdir=/tmp/tidb_cdc_test/row_format + cluster_type=normal + cd /tmp/tidb_cdc_test/row_format + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/row_format/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/row_format/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/row_format/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/row_format/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/row_format/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/row_format/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/row_format/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/row_format/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/row_format/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:08:44 CST 2024] <<<<<< START pulsar cluster in normal mode in row_format case >>>>>>' [Sat May 4 11:08:44 CST 2024] <<<<<< START pulsar cluster in normal mode in row_format case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/row_format/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/row_format/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/row_format/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Sat May 4 11:08:50 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:08:50 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.3340.out cli changefeed create --start-ts=449522465496104961 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-row-format-test-27510?protocol=canal-json&enable-tidb-extension=true' + /usr/local/pulsar/bin/pulsar-admin namespaces list public Create changefeed successfully! ID: 41bf9b1f-060d-40a5-8cea-5f6f68f7dd66 Info: {"upstream_id":7364976014090214956,"namespace":"default","id":"41bf9b1f-060d-40a5-8cea-5f6f68f7dd66","sink_uri":"pulsar://127.0.0.1:6650/ticdc-row-format-test-27510?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:08:51.852610474+08:00","start_ts":449522465496104961,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522465496104961,"checkpoint_ts":449522465496104961,"checkpoint_time":"2024-05-04 11:08:39.965"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... public/default + set +x [Sat May 4 11:08:53 CST 2024] <<<<<< START Pulsar consumer in row_format case >>>>>> ++ date + echo '[Sat May 4 11:08:53 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:08:53 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.3419.out cli changefeed create --start-ts=449522464985972737 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-foreign-key-test-12299?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: a4956346-73e0-47a8-9b9d-e73b4f0c052e Info: {"upstream_id":7364976018398550598,"namespace":"default","id":"a4956346-73e0-47a8-9b9d-e73b4f0c052e","sink_uri":"pulsar://127.0.0.1:6650/ticdc-foreign-key-test-12299?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:08:54.396895922+08:00","start_ts":449522464985972737,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522464985972737,"checkpoint_ts":449522464985972737,"checkpoint_time":"2024-05-04 11:08:38.019"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x [Sat May 4 11:08:56 CST 2024] <<<<<< START Pulsar consumer in foreign_key case >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 16.14 secs (231059403 bytes/sec) [Pipeline] { [Pipeline] cache table foreign_key.finish_mark not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 5-th check, retry later table row_format.finish_mark not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 6-th check, retry later table foreign_key.finish_mark not exists for 7-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later table row_format.finish_mark not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 8-th check, retry later table row_format.finish_mark not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 9-th check, retry later table row_format.finish_mark not exists for 5-th check, retry later table foreign_key.finish_mark not exists for 10-th check, retry later table row_format.finish_mark not exists for 6-th check, retry later table foreign_key.finish_mark not exists for 11-th check, retry later table foreign_key.finish_mark exists check diff successfully table row_format.finish_mark not exists for 7-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 21.18 secs (176092790 bytes/sec) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:09:22 CST 2024] <<<<<< run test case foreign_key success! >>>>>> [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 8-th check, retry later table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:09:26 CST 2024] <<<<<< run test case row_format success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 6.75 secs (552975867 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0650ed7c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:4699, start at 2024-05-04 11:09:44.681297207 +0800 CST m=+5.080576633 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:11:44.690 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:09:44.671 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:59:44.671 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0650ed7c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:4699, start at 2024-05-04 11:09:44.681297207 +0800 CST m=+5.080576633 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:11:44.690 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:09:44.671 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:59:44.671 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0650efc80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:4775, start at 2024-05-04 11:09:44.847788911 +0800 CST m=+5.195241540 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:11:44.855 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:09:44.818 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:59:44.818 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 14.70 secs (253783543 bytes/sec) [Pipeline] { [Pipeline] cache ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.6163.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449522483389005825 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522483389005825 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:09:49 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.62146216.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06515f6c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:4657, start at 2024-05-04 11:09:51.975253928 +0800 CST m=+5.510406758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:11:51.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:09:51.963 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:59:51.963 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:09:52 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/075dffae-ea67-45cb-9a72-67a8995c3796 {"id":"075dffae-ea67-45cb-9a72-67a8995c3796","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792190} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41941576e9 075dffae-ea67-45cb-9a72-67a8995c3796 /tidb/cdc/default/default/upstream/7364976314820718075 {"id":7364976314820718075,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/075dffae-ea67-45cb-9a72-67a8995c3796 {"id":"075dffae-ea67-45cb-9a72-67a8995c3796","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792190} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41941576e9 075dffae-ea67-45cb-9a72-67a8995c3796 /tidb/cdc/default/default/upstream/7364976314820718075 {"id":7364976314820718075,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/075dffae-ea67-45cb-9a72-67a8995c3796 {"id":"075dffae-ea67-45cb-9a72-67a8995c3796","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792190} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41941576e9 075dffae-ea67-45cb-9a72-67a8995c3796 /tidb/cdc/default/default/upstream/7364976314820718075 {"id":7364976314820718075,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:09:52 CST 2024] <<<<<< START pulsar cluster in ddl_puller_lag case >>>>>>' [Sat May 4 11:09:52 CST 2024] <<<<<< START pulsar cluster in ddl_puller_lag case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_puller_lag + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_puller_lag + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_puller_lag/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_puller_lag/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_puller_lag/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:09:52 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_puller_lag case >>>>>>' [Sat May 4 11:09:52 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_puller_lag case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_puller_lag/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_puller_lag/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_puller_lag/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06515f6c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:4657, start at 2024-05-04 11:09:51.975253928 +0800 CST m=+5.510406758 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:11:51.981 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:09:51.963 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:59:51.963 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06515c2c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:4736, start at 2024-05-04 11:09:51.778015467 +0800 CST m=+5.262693204 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:11:51.785 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:09:51.755 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-10:59:51.755 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 6.21 secs (601057575 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=02474972-92ef-4559-8613-e266001f6c47 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-0hqq-5dxcz HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-0hqqd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-0hqqd pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-0hqq-5dxcz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-0hqq-5dxcz GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:09:56 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.6134.out cli tso query --pd=http://127.0.0.1:2379 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=f5f2fcd7-ccfa-47f9-aa1b-1b00c862593b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-sbb7p GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35 pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-sbb7p GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=2d23f820-e3a0-4f2f-8f51-c7aa33e35aae BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-blcq2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89 pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-blcq2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=a0c0bda1-3ff6-4cb8-a321-f34822e18648 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-7r7sz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-7r7sz pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=a67d4c69-67e0-4123-838b-3cebe424a74f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-nrk1j GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-nrk1j pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=79a0833f-5213-41b2-98cf-c0b2be8f8121 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-j8s10 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5 pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-j8s10 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... public/default ++ date + echo '[Sat May 4 11:09:59 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:09:59 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.6678.out cli changefeed create --start-ts=449522483389005825 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-20012?protocol=canal-json&enable-tidb-extension=true' + set +x + tso='449522485877014529 PASS find: '/tmp/tidb_cdc_test/*/*': No such file or directory coverage: 1.8% of statements in github.com/pingcap/tiflow/...' =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: pulsar... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused + echo 449522485877014529 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 [Sat May 4 11:09:59 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.61726174.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // timeout [Pipeline] } Create changefeed successfully! ID: c3fa07ee-8633-4530-8cf6-8b994ac81d82 Info: {"upstream_id":7364976314820718075,"namespace":"default","id":"c3fa07ee-8633-4530-8cf6-8b994ac81d82","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-20012?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:10:00.151459806+08:00","start_ts":449522483389005825,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522483389005825,"checkpoint_ts":449522483389005825,"checkpoint_time":"2024-05-04 11:09:48.221"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide + set +x [Sat May 4 11:10:01 CST 2024] <<<<<< START Pulsar consumer in ddl_puller_lag case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:10:02 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24f1abd3-ece7-4a8e-91e8-c456901469b8 {"id":"24f1abd3-ece7-4a8e-91e8-c456901469b8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41942c76d8 24f1abd3-ece7-4a8e-91e8-c456901469b8 /tidb/cdc/default/default/upstream/7364976346155453192 {"id":7364976346155453192,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24f1abd3-ece7-4a8e-91e8-c456901469b8 {"id":"24f1abd3-ece7-4a8e-91e8-c456901469b8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41942c76d8 24f1abd3-ece7-4a8e-91e8-c456901469b8 /tidb/cdc/default/default/upstream/7364976346155453192 {"id":7364976346155453192,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/24f1abd3-ece7-4a8e-91e8-c456901469b8 {"id":"24f1abd3-ece7-4a8e-91e8-c456901469b8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792199} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41942c76d8 24f1abd3-ece7-4a8e-91e8-c456901469b8 /tidb/cdc/default/default/upstream/7364976346155453192 {"id":7364976346155453192,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:10:02 CST 2024] <<<<<< START pulsar cluster in drop_many_tables case >>>>>>' [Sat May 4 11:10:02 CST 2024] <<<<<< START pulsar cluster in drop_many_tables case >>>>>> + workdir=/tmp/tidb_cdc_test/drop_many_tables + cluster_type=normal + cd /tmp/tidb_cdc_test/drop_many_tables + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/drop_many_tables/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/drop_many_tables/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/drop_many_tables/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/drop_many_tables/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/drop_many_tables/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/drop_many_tables/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/drop_many_tables/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/drop_many_tables/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/drop_many_tables/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/drop_many_tables/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/drop_many_tables/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/drop_many_tables/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:10:02 CST 2024] <<<<<< START pulsar cluster in normal mode in drop_many_tables case >>>>>>' [Sat May 4 11:10:02 CST 2024] <<<<<< START pulsar cluster in normal mode in drop_many_tables case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/drop_many_tables/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/drop_many_tables/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/drop_many_tables/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC2A6C24D68986 < X-Xss-Protection: 1; mode=block < Date: Sat, 04 May 2024 03:10:02 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Sat May 4 11:10:02 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] // stage [Pipeline] } [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] // container [Pipeline] } [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] timeout Timeout set to expire in 40 min + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] // withEnv [Pipeline] } [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:05 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) public/default ++ date + echo '[Sat May 4 11:10:08 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:10:08 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.6638.out cli changefeed create --start-ts=449522485877014529 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-drop-tables-test-5070?protocol=canal-json&enable-tidb-extension=true' Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:08 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 7a7c6a59-96fb-4893-bf35-42a626c4a5d1 Info: {"upstream_id":7364976346155453192,"namespace":"default","id":"7a7c6a59-96fb-4893-bf35-42a626c4a5d1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-drop-tables-test-5070?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:10:09.699055269+08:00","start_ts":449522485877014529,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522485877014529,"checkpoint_ts":449522485877014529,"checkpoint_time":"2024-05-04 11:09:57.712"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Sat May 4 11:10:11 CST 2024] <<<<<< START Pulsar consumer in drop_many_tables case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:11 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652a2640011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89, pid:1306, start at 2024-05-04 11:10:12.652171329 +0800 CST m=+5.113283521 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:12.658 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:12.633 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:12.633 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652a2640011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89, pid:1306, start at 2024-05-04 11:10:12.652171329 +0800 CST m=+5.113283521 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:12.658 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:12.633 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:12.633 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652b1e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:1356, start at 2024-05-04 11:10:13.630317123 +0800 CST m=+5.163757176 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:13.637 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:13.626 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:13.626 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: pulsar... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652b0b4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:1300, start at 2024-05-04 11:10:13.559515085 +0800 CST m=+5.231245872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:13.566 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:13.549 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:13.549 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652a1d40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:1360, start at 2024-05-04 11:10:12.604769835 +0800 CST m=+5.199409649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:12.610 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:12.597 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:12.597 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652a1d40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:1360, start at 2024-05-04 11:10:12.604769835 +0800 CST m=+5.199409649 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:12.610 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:12.597 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:12.597 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652a2ac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:1437, start at 2024-05-04 11:10:12.692455138 +0800 CST m=+5.232579242 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:12.698 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:12.701 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:12.701 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table drop_tables.c not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652c5bc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35, pid:1362, start at 2024-05-04 11:10:14.913192115 +0800 CST m=+5.076454592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:14.918 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:14.895 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:14.895 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652c5bc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35, pid:1362, start at 2024-05-04 11:10:14.913192115 +0800 CST m=+5.076454592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:14.918 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:14.895 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:14.895 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652c7f40001 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35, pid:1451, start at 2024-05-04 11:10:15.037936026 +0800 CST m=+5.145032180 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:15.046 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:15.037 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:15.037 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652b514000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89, pid:1382, start at 2024-05-04 11:10:13.841295408 +0800 CST m=+6.249288389 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:13.847 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:13.829 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:13.829 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652b0b4000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:1300, start at 2024-05-04 11:10:13.559515085 +0800 CST m=+5.231245872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:13.566 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:13.549 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:13.549 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652b2fc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:1390, start at 2024-05-04 11:10:13.723943335 +0800 CST m=+5.335661700 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:13.732 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:13.695 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:13.695 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC2A6F8DE23F89 < X-Xss-Protection: 1; mode=block < Date: Sat, 04 May 2024 03:10:16 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Sat May 4 11:10:16 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652b1e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:1356, start at 2024-05-04 11:10:13.630317123 +0800 CST m=+5.163757176 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:13.637 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:13.626 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:13.626 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0652b29c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:1441, start at 2024-05-04 11:10:13.707786855 +0800 CST m=+5.184612053 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:13.714 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:13.671 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:13.671 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ++ date + echo '[Sat May 4 11:10:17 CST 2024] <<<<<< START pulsar cluster in kv_client_stream_reconnect case >>>>>>' [Sat May 4 11:10:17 CST 2024] <<<<<< START pulsar cluster in kv_client_stream_reconnect case >>>>>> + workdir=/tmp/tidb_cdc_test/kv_client_stream_reconnect + cluster_type=oauth + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_standalone.conf + pulsar_port=6650 + '[' oauth == mtls ']' + '[' oauth == oauth ']' + echo 'Waiting for oauth2 server to be ready...' Waiting for oauth2 server to be ready... + i=0 + nc -z localhost 9096 + oauth2-server + gen_oauth_config + echo ' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ date + echo '[Sat May 4 11:10:17 CST 2024] <<<<<< START pulsar cluster in oauth mode in kv_client_stream_reconnect case >>>>>>' [Sat May 4 11:10:17 CST 2024] <<<<<< START pulsar cluster in oauth mode in kv_client_stream_reconnect case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2843.out cli tso query --pd=http://127.0.0.1:2379 table drop_tables.c not exists for 4-th check, retry later [Sat May 4 11:10:18 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.28842886.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ date + echo '[Sat May 4 11:10:18 CST 2024] <<<<<< START pulsar cluster in gc_safepoint case >>>>>>' [Sat May 4 11:10:18 CST 2024] <<<<<< START pulsar cluster in gc_safepoint case >>>>>> + workdir=/tmp/tidb_cdc_test/gc_safepoint + cluster_type=normal + cd /tmp/tidb_cdc_test/gc_safepoint + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/gc_safepoint/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/gc_safepoint/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/gc_safepoint/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/gc_safepoint/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/gc_safepoint/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/gc_safepoint/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/gc_safepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/gc_safepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/gc_safepoint/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:10:18 CST 2024] <<<<<< START pulsar cluster in normal mode in gc_safepoint case >>>>>>' [Sat May 4 11:10:18 CST 2024] <<<<<< START pulsar cluster in normal mode in gc_safepoint case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/gc_safepoint/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/gc_safepoint/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/gc_safepoint/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ++ date + echo '[Sat May 4 11:10:18 CST 2024] <<<<<< START pulsar cluster in processor_err_chan case >>>>>>' [Sat May 4 11:10:18 CST 2024] <<<<<< START pulsar cluster in processor_err_chan case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_err_chan + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_err_chan + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_err_chan/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_err_chan/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_err_chan/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_err_chan/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_err_chan/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_err_chan/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_err_chan/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_err_chan/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_err_chan/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:10:18 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_err_chan case >>>>>>' [Sat May 4 11:10:18 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_err_chan case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_err_chan/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_err_chan/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_err_chan/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + set +x + tso='449522491207974913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522491207974913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:19 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c exists check diff successfully wait process cdc.test exit for 1-th time... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ***************** properties ***************** "readproportion"="0" "mysql.port"="4000" "insertproportion"="0" "threadcount"="2" "dotransactions"="false" "workload"="core" "updateproportion"="0" "scanproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "operationcount"="0" "recordcount"="10" "mysql.db"="sink_retry" "mysql.user"="root" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 8.562995ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2054.9, Avg(us): 1634, Min(us): 1010, Max(us): 3647, 95th(us): 4000, 99th(us): 4000 [Sat May 4 11:10:20 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.29062908.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:10:21 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:10:21 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a9c3c54-ecfd-40e9-8a44-1997c5d46596 {"id":"3a9c3c54-ecfd-40e9-8a44-1997c5d46596","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792218} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419485dbc6 3a9c3c54-ecfd-40e9-8a44-1997c5d46596 /tidb/cdc/default/default/upstream/7364976446179841566 {"id":7364976446179841566,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a9c3c54-ecfd-40e9-8a44-1997c5d46596 {"id":"3a9c3c54-ecfd-40e9-8a44-1997c5d46596","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792218} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419485dbc6 3a9c3c54-ecfd-40e9-8a44-1997c5d46596 /tidb/cdc/default/default/upstream/7364976446179841566 {"id":7364976446179841566,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3a9c3c54-ecfd-40e9-8a44-1997c5d46596 {"id":"3a9c3c54-ecfd-40e9-8a44-1997c5d46596","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792218} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419485dbc6 3a9c3c54-ecfd-40e9-8a44-1997c5d46596 /tidb/cdc/default/default/upstream/7364976446179841566 {"id":7364976446179841566,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:10:21 CST 2024] <<<<<< START pulsar cluster in charset_gbk case >>>>>>' [Sat May 4 11:10:21 CST 2024] <<<<<< START pulsar cluster in charset_gbk case >>>>>> + workdir=/tmp/tidb_cdc_test/charset_gbk + cluster_type=mtls + cd /tmp/tidb_cdc_test/charset_gbk + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/charset_gbk/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/charset_gbk/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/charset_gbk/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/charset_gbk/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/charset_gbk/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus ........................................................................................................+++ ..................................................+++ e is 65537 (0x10001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus ...................................................+++ ...................................+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=/CN=server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus .........+++ .......................+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=/CN=broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/charset_gbk/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/charset_gbk/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem' ++ date + echo '[Sat May 4 11:10:21 CST 2024] <<<<<< START pulsar cluster in mtls mode in charset_gbk case >>>>>>' [Sat May 4 11:10:21 CST 2024] <<<<<< START pulsar cluster in mtls mode in charset_gbk case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/charset_gbk/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/charset_gbk/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/charset_gbk/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:23 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:10:23 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b1e25ae-bba6-4d89-a152-bb559bc2b6c9 {"id":"9b1e25ae-bba6-4d89-a152-bb559bc2b6c9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792220} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41947e42f4 9b1e25ae-bba6-4d89-a152-bb559bc2b6c9 /tidb/cdc/default/default/upstream/7364976438110253596 {"id":7364976438110253596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b1e25ae-bba6-4d89-a152-bb559bc2b6c9 {"id":"9b1e25ae-bba6-4d89-a152-bb559bc2b6c9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792220} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41947e42f4 9b1e25ae-bba6-4d89-a152-bb559bc2b6c9 /tidb/cdc/default/default/upstream/7364976438110253596 {"id":7364976438110253596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b1e25ae-bba6-4d89-a152-bb559bc2b6c9 {"id":"9b1e25ae-bba6-4d89-a152-bb559bc2b6c9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792220} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41947e42f4 9b1e25ae-bba6-4d89-a152-bb559bc2b6c9 /tidb/cdc/default/default/upstream/7364976438110253596 {"id":7364976438110253596,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:10:23 CST 2024] <<<<<< START pulsar cluster in sink_retry case >>>>>>' [Sat May 4 11:10:23 CST 2024] <<<<<< START pulsar cluster in sink_retry case >>>>>> + workdir=/tmp/tidb_cdc_test/sink_retry + cluster_type=normal + cd /tmp/tidb_cdc_test/sink_retry + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/sink_retry/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/sink_retry/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/sink_retry/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/sink_retry/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/sink_retry/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/sink_retry/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/sink_retry/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/sink_retry/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/sink_retry/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:10:23 CST 2024] <<<<<< START pulsar cluster in normal mode in sink_retry case >>>>>>' [Sat May 4 11:10:23 CST 2024] <<<<<< START pulsar cluster in normal mode in sink_retry case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/sink_retry/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/sink_retry/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/sink_retry/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Sat May 4 11:10:26 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:10:26 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 11:10:26 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.32573259.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 public/default public/default ++ date + echo '[Sat May 4 11:10:26 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:10:26 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 11:10:26 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.32433245.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:26 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> ++ date + echo '[Sat May 4 11:10:26 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:10:26 CST 2024] <<<<<< pulsar is ready >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 20.42 secs (182645961 bytes/sec) [Pipeline] { [Pipeline] cache + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Sat May 4 11:10:29 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.33883390.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:10:29 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/132504a8-97f7-4ada-968c-b68585a07923 {"id":"132504a8-97f7-4ada-968c-b68585a07923","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792226} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419480bbf3 132504a8-97f7-4ada-968c-b68585a07923 /tidb/cdc/default/default/upstream/7364976436807092256 {"id":7364976436807092256,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/132504a8-97f7-4ada-968c-b68585a07923 {"id":"132504a8-97f7-4ada-968c-b68585a07923","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792226} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419480bbf3 132504a8-97f7-4ada-968c-b68585a07923 /tidb/cdc/default/default/upstream/7364976436807092256 {"id":7364976436807092256,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/132504a8-97f7-4ada-968c-b68585a07923 {"id":"132504a8-97f7-4ada-968c-b68585a07923","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792226} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419480bbf3 132504a8-97f7-4ada-968c-b68585a07923 /tidb/cdc/default/default/upstream/7364976436807092256 {"id":7364976436807092256,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x public/default + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:10:29 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b933718f-dea2-49aa-b7ee-494f364f2525 {"id":"b933718f-dea2-49aa-b7ee-494f364f2525","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792227} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4194846aee b933718f-dea2-49aa-b7ee-494f364f2525 /tidb/cdc/default/default/upstream/7364976441465067820 {"id":7364976441465067820,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b933718f-dea2-49aa-b7ee-494f364f2525 {"id":"b933718f-dea2-49aa-b7ee-494f364f2525","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792227} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4194846aee b933718f-dea2-49aa-b7ee-494f364f2525 /tidb/cdc/default/default/upstream/7364976441465067820 {"id":7364976441465067820,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b933718f-dea2-49aa-b7ee-494f364f2525 {"id":"b933718f-dea2-49aa-b7ee-494f364f2525","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792227} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4194846aee b933718f-dea2-49aa-b7ee-494f364f2525 /tidb/cdc/default/default/upstream/7364976441465067820 {"id":7364976441465067820,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:10:30 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:10:30 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.3372.out cli changefeed create --start-ts=449522491207974913 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-29925?protocol=canal-json&enable-tidb-extension=true' find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:29 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> [Sat May 4 11:10:30 CST 2024] <<<<<< START Pulsar consumer in kv_client_stream_reconnect case >>>>>> [Sat May 4 11:10:30 CST 2024] <<<<<< START Pulsar consumer in gc_safepoint case >>>>>> 0 check diff failed 1-th time, retry later Create changefeed successfully! ID: c4b2a5bd-e018-4cce-96e7-6cc17f619a58 Info: {"upstream_id":7364976438110253596,"namespace":"default","id":"c4b2a5bd-e018-4cce-96e7-6cc17f619a58","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-29925?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:10:30.976923464+08:00","start_ts":449522491207974913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522491207974913,"checkpoint_ts":449522491207974913,"checkpoint_time":"2024-05-04 11:10:18.048"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... public/default ++ date + echo '[Sat May 4 11:10:30 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:10:30 CST 2024] <<<<<< pulsar is ready >>>>>> Create changefeed successfully! ID: db462766-0293-41c5-9bc4-fae344e46e34 Info: {"upstream_id":7364976446179841566,"namespace":"default","id":"db462766-0293-41c5-9bc4-fae344e46e34","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T11:10:31.114364812+08:00","start_ts":449522491259879425,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522491259879425,"checkpoint_ts":449522491259879425,"checkpoint_time":"2024-05-04 11:10:18.246"} [Sat May 4 11:10:31 CST 2024] <<<<<< START Pulsar consumer in charset_gbk case >>>>>> + set +x [Sat May 4 11:10:32 CST 2024] <<<<<< START Pulsar consumer in sink_retry case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:10:32 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c154a98f-cb34-477c-b66c-696528183b05 {"id":"c154a98f-cb34-477c-b66c-696528183b05","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792229} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419481c58c c154a98f-cb34-477c-b66c-696528183b05 /tidb/cdc/default/default/upstream/7364976438259205069 {"id":7364976438259205069,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c154a98f-cb34-477c-b66c-696528183b05 {"id":"c154a98f-cb34-477c-b66c-696528183b05","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792229} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419481c58c c154a98f-cb34-477c-b66c-696528183b05 /tidb/cdc/default/default/upstream/7364976438259205069 {"id":7364976438259205069,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c154a98f-cb34-477c-b66c-696528183b05 {"id":"c154a98f-cb34-477c-b66c-696528183b05","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792229} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419481c58c c154a98f-cb34-477c-b66c-696528183b05 /tidb/cdc/default/default/upstream/7364976438259205069 {"id":7364976438259205069,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:10:33 CST 2024] <<<<<< START Pulsar consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 31adb4b7-ef7d-4cb8-af6d-853e361d8b32 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=31adb4b7-ef7d-4cb8-af6d-853e361d8b32 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 31adb4b7-ef7d-4cb8-af6d-853e361d8b32 -s + info='{ "upstream_id": 7364976438259205069, "namespace": "default", "id": "31adb4b7-ef7d-4cb8-af6d-853e361d8b32", "state": "normal", "checkpoint_tso": 449522494956634114, "checkpoint_time": "2024-05-04 11:10:32.348", "error": null }' + echo '{ "upstream_id": 7364976438259205069, "namespace": "default", "id": "31adb4b7-ef7d-4cb8-af6d-853e361d8b32", "state": "normal", "checkpoint_tso": 449522494956634114, "checkpoint_time": "2024-05-04 11:10:32.348", "error": null }' { "upstream_id": 7364976438259205069, "namespace": "default", "id": "31adb4b7-ef7d-4cb8-af6d-853e361d8b32", "state": "normal", "checkpoint_tso": 449522494956634114, "checkpoint_time": "2024-05-04 11:10:32.348", "error": null } ++ echo '{' '"upstream_id":' 7364976438259205069, '"namespace":' '"default",' '"id":' '"31adb4b7-ef7d-4cb8-af6d-853e361d8b32",' '"state":' '"normal",' '"checkpoint_tso":' 449522494956634114, '"checkpoint_time":' '"2024-05-04' '11:10:32.348",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364976438259205069, '"namespace":' '"default",' '"id":' '"31adb4b7-ef7d-4cb8-af6d-853e361d8b32",' '"state":' '"normal",' '"checkpoint_tso":' 449522494956634114, '"checkpoint_time":' '"2024-05-04' '11:10:32.348",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run pulsar + '[' pulsar '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Sat May 4 11:10:32 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Sat May 4 11:10:32 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster check diff failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... check diff failed 1-th time, retry later check diff failed 3-th time, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:35 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> check diff failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:39 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> check diff successfully check_safepoint_forward http://127.0.0.1:2379 7364976441465067820 449522496051871748 449522494308614149 check diff successfully wait process cdc.test exit for 1-th time... table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:10:40 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> run task successfully check_changefeed_state http://127.0.0.1:2379 b02c283e-f300-4e3d-8752-ba48a566e7b2 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=b02c283e-f300-4e3d-8752-ba48a566e7b2 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b02c283e-f300-4e3d-8752-ba48a566e7b2 -s + info='{ "upstream_id": 7364976441465067820, "namespace": "default", "id": "b02c283e-f300-4e3d-8752-ba48a566e7b2", "state": "stopped", "checkpoint_tso": 449522496838041606, "checkpoint_time": "2024-05-04 11:10:39.525", "error": null }' + echo '{ "upstream_id": 7364976441465067820, "namespace": "default", "id": "b02c283e-f300-4e3d-8752-ba48a566e7b2", "state": "stopped", "checkpoint_tso": 449522496838041606, "checkpoint_time": "2024-05-04 11:10:39.525", "error": null }' { "upstream_id": 7364976441465067820, "namespace": "default", "id": "b02c283e-f300-4e3d-8752-ba48a566e7b2", "state": "stopped", "checkpoint_tso": 449522496838041606, "checkpoint_time": "2024-05-04 11:10:39.525", "error": null } ++ echo '{' '"upstream_id":' 7364976441465067820, '"namespace":' '"default",' '"id":' '"b02c283e-f300-4e3d-8752-ba48a566e7b2",' '"state":' '"stopped",' '"checkpoint_tso":' 449522496838041606, '"checkpoint_time":' '"2024-05-04' '11:10:39.525",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364976441465067820, '"namespace":' '"default",' '"id":' '"b02c283e-f300-4e3d-8752-ba48a566e7b2",' '"state":' '"stopped",' '"checkpoint_tso":' 449522496838041606, '"checkpoint_time":' '"2024-05-04' '11:10:39.525",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7364976441465067820 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.finish_mark not exists for 2-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:10:42 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 12.69 secs (293967948 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully check_changefeed_state http://127.0.0.1:2379 b02c283e-f300-4e3d-8752-ba48a566e7b2 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=b02c283e-f300-4e3d-8752-ba48a566e7b2 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b02c283e-f300-4e3d-8752-ba48a566e7b2 -s + info='{ "upstream_id": 7364976441465067820, "namespace": "default", "id": "b02c283e-f300-4e3d-8752-ba48a566e7b2", "state": "normal", "checkpoint_tso": 449522496838041606, "checkpoint_time": "2024-05-04 11:10:39.525", "error": null }' + echo '{ "upstream_id": 7364976441465067820, "namespace": "default", "id": "b02c283e-f300-4e3d-8752-ba48a566e7b2", "state": "normal", "checkpoint_tso": 449522496838041606, "checkpoint_time": "2024-05-04 11:10:39.525", "error": null }' { "upstream_id": 7364976441465067820, "namespace": "default", "id": "b02c283e-f300-4e3d-8752-ba48a566e7b2", "state": "normal", "checkpoint_tso": 449522496838041606, "checkpoint_time": "2024-05-04 11:10:39.525", "error": null } ++ echo '{' '"upstream_id":' 7364976441465067820, '"namespace":' '"default",' '"id":' '"b02c283e-f300-4e3d-8752-ba48a566e7b2",' '"state":' '"normal",' '"checkpoint_tso":' 449522496838041606, '"checkpoint_time":' '"2024-05-04' '11:10:39.525",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364976441465067820, '"namespace":' '"default",' '"id":' '"b02c283e-f300-4e3d-8752-ba48a566e7b2",' '"state":' '"normal",' '"checkpoint_tso":' 449522496838041606, '"checkpoint_time":' '"2024-05-04' '11:10:39.525",' '"error":' null '}' ++ jq -r .error.message table test.finish_mark not exists for 3-th check, retry later + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7364976441465067820 449522496838041605 449522496838041606 run task successfully \033[0;36m<<< Run all test success >>>\033[0m check_changefeed_state http://127.0.0.1:2379 b02c283e-f300-4e3d-8752-ba48a566e7b2 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=b02c283e-f300-4e3d-8752-ba48a566e7b2 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c b02c283e-f300-4e3d-8752-ba48a566e7b2 -s + info='{ "upstream_id": 7364976441465067820, "namespace": "default", "id": "b02c283e-f300-4e3d-8752-ba48a566e7b2", "state": "stopped", "checkpoint_tso": 449522496838041606, "checkpoint_time": "2024-05-04 11:10:39.525", "error": null }' + echo '{ "upstream_id": 7364976441465067820, "namespace": "default", "id": "b02c283e-f300-4e3d-8752-ba48a566e7b2", "state": "stopped", "checkpoint_tso": 449522496838041606, "checkpoint_time": "2024-05-04 11:10:39.525", "error": null }' { "upstream_id": 7364976441465067820, "namespace": "default", "id": "b02c283e-f300-4e3d-8752-ba48a566e7b2", "state": "stopped", "checkpoint_tso": 449522496838041606, "checkpoint_time": "2024-05-04 11:10:39.525", "error": null } ++ echo '{' '"upstream_id":' 7364976441465067820, '"namespace":' '"default",' '"id":' '"b02c283e-f300-4e3d-8752-ba48a566e7b2",' '"state":' '"stopped",' '"checkpoint_tso":' 449522496838041606, '"checkpoint_time":' '"2024-05-04' '11:10:39.525",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364976441465067820, '"namespace":' '"default",' '"id":' '"b02c283e-f300-4e3d-8752-ba48a566e7b2",' '"state":' '"stopped",' '"checkpoint_tso":' 449522496838041606, '"checkpoint_time":' '"2024-05-04' '11:10:39.525",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 570feb5e-5575-475d-8be1-57d1b766a0af normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=570feb5e-5575-475d-8be1-57d1b766a0af + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 570feb5e-5575-475d-8be1-57d1b766a0af -s + info='{ "upstream_id": 7364976441465067820, "namespace": "default", "id": "570feb5e-5575-475d-8be1-57d1b766a0af", "state": "normal", "checkpoint_tso": 449522498581299201, "checkpoint_time": "2024-05-04 11:10:46.175", "error": null }' + echo '{ "upstream_id": 7364976441465067820, "namespace": "default", "id": "570feb5e-5575-475d-8be1-57d1b766a0af", "state": "normal", "checkpoint_tso": 449522498581299201, "checkpoint_time": "2024-05-04 11:10:46.175", "error": null }' { "upstream_id": 7364976441465067820, "namespace": "default", "id": "570feb5e-5575-475d-8be1-57d1b766a0af", "state": "normal", "checkpoint_tso": 449522498581299201, "checkpoint_time": "2024-05-04 11:10:46.175", "error": null } ++ echo '{' '"upstream_id":' 7364976441465067820, '"namespace":' '"default",' '"id":' '"570feb5e-5575-475d-8be1-57d1b766a0af",' '"state":' '"normal",' '"checkpoint_tso":' 449522498581299201, '"checkpoint_time":' '"2024-05-04' '11:10:46.175",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364976441465067820, '"namespace":' '"default",' '"id":' '"570feb5e-5575-475d-8be1-57d1b766a0af",' '"state":' '"normal",' '"checkpoint_tso":' 449522498581299201, '"checkpoint_time":' '"2024-05-04' '11:10:46.175",' '"error":' null '}' ++ jq -r .error.message table test.finish_mark not exists for 4-th check, retry later + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7364976441465067820 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.finish_mark not exists for 5-th check, retry later run task successfully Changefeed remove successfully. ID: b02c283e-f300-4e3d-8752-ba48a566e7b2 CheckpointTs: 449522496838041606 SinkURI: pulsar://127.0.0.1:6650/ticdc-gc-safepoint-31009?protocol=canal-json&enable-tidb-extension=true VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0654d8bc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:7868, start at 2024-05-04 11:10:48.926776836 +0800 CST m=+5.309546012 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:48.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:48.929 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:48.929 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_safepoint_forward http://127.0.0.1:2379 7364976441465067820 449522499459743748 449522498581299201 449522496838041606 table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... run task successfully wait process cdc.test exit for 2-th time... Changefeed remove successfully. ID: 570feb5e-5575-475d-8be1-57d1b766a0af CheckpointTs: 449522499721625606 SinkURI: pulsar://127.0.0.1:6650/ticdc-gc-safepoint-31009?protocol=canal-json&enable-tidb-extension=true check_safepoint_cleared http://127.0.0.1:2379 7364976441465067820 run task successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0654d8bc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:7868, start at 2024-05-04 11:10:48.926776836 +0800 CST m=+5.309546012 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:48.933 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:48.929 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:48.929 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0654dd5c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:7955, start at 2024-05-04 11:10:49.195919726 +0800 CST m=+5.524078604 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:12:49.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:10:49.175 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:00:49.175 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:10:52 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:10:53 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> ++ date + echo '[Sat May 4 11:10:53 CST 2024] <<<<<< START pulsar cluster in processor_stop_delay case >>>>>>' [Sat May 4 11:10:53 CST 2024] <<<<<< START pulsar cluster in processor_stop_delay case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_stop_delay + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_stop_delay + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_stop_delay/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_stop_delay/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_stop_delay/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_stop_delay/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_stop_delay/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_stop_delay/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_stop_delay/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_stop_delay/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_stop_delay/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_stop_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_stop_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_stop_delay/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:10:53 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_stop_delay case >>>>>>' [Sat May 4 11:10:53 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_stop_delay case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_stop_delay/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_stop_delay/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_stop_delay/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release public/default =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ++ date + echo '[Sat May 4 11:11:02 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:11:02 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 11:11:02 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.97789780.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "readallfields"="true" "mysql.user"="root" "readproportion"="0" "updateproportion"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.db"="sink_retry" "dotransactions"="false" "operationcount"="0" "insertproportion"="0" "workload"="core" "mysql.port"="4000" "recordcount"="10" "requestdistribution"="uniform" ********************************************** Run finished, takes 27.017689ms INSERT - Takes(s): 0.0, Count: 10, OPS: 389.6, Avg(us): 5121, Min(us): 566, Max(us): 23116, 95th(us): 24000, 99th(us): 24000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:05 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cac2721-6024-48cc-a339-a4640d212025 {"id":"9cac2721-6024-48cc-a339-a4640d212025","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792262} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419509d4f0 9cac2721-6024-48cc-a339-a4640d212025 /tidb/cdc/default/default/upstream/7364976592046983906 {"id":7364976592046983906,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cac2721-6024-48cc-a339-a4640d212025 {"id":"9cac2721-6024-48cc-a339-a4640d212025","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792262} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419509d4f0 9cac2721-6024-48cc-a339-a4640d212025 /tidb/cdc/default/default/upstream/7364976592046983906 {"id":7364976592046983906,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9cac2721-6024-48cc-a339-a4640d212025 {"id":"9cac2721-6024-48cc-a339-a4640d212025","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792262} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419509d4f0 9cac2721-6024-48cc-a339-a4640d212025 /tidb/cdc/default/default/upstream/7364976592046983906 {"id":7364976592046983906,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 20.49 secs (182081180 bytes/sec) [Pipeline] { [Pipeline] cache [Sat May 4 11:11:05 CST 2024] <<<<<< START Pulsar consumer in processor_stop_delay case >>>>>> table processor_stop_delay.t not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table processor_stop_delay.t exists check diff failed 1-th time, retry later Verifying downstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0655fb5c001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:4864, start at 2024-05-04 11:11:07.522937816 +0800 CST m=+5.155184216 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:07.529 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:07.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:07.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0655fb5c001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:4864, start at 2024-05-04 11:11:07.522937816 +0800 CST m=+5.155184216 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:07.529 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:07.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:07.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0655fe5c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:4949, start at 2024-05-04 11:11:07.685905824 +0800 CST m=+5.252542663 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:07.693 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:07.671 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:07.671 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ++ date + echo '[Sat May 4 11:11:10 CST 2024] <<<<<< START pulsar cluster in changefeed_reconstruct case >>>>>>' [Sat May 4 11:11:10 CST 2024] <<<<<< START pulsar cluster in changefeed_reconstruct case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_reconstruct + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_reconstruct + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_reconstruct/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:11:10 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_reconstruct case >>>>>>' [Sat May 4 11:11:10 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_reconstruct case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 check diff successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:11:10 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 2-th time, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 7.60 secs (490822660 bytes/sec) [Pipeline] { Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) public/default ++ date + echo '[Sat May 4 11:11:18 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:11:18 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 11:11:18 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.67466748.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0656a270000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35, pid:4539, start at 2024-05-04 11:11:18.181644904 +0800 CST m=+5.140673296 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:18.188 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:18.172 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:18.172 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0656a270000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35, pid:4539, start at 2024-05-04 11:11:18.181644904 +0800 CST m=+5.140673296 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:18.188 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:18.172 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:18.172 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0656a67c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-sbb7-5mn35, pid:4622, start at 2024-05-04 11:11:18.451975602 +0800 CST m=+5.349913392 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:18.460 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:18.431 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:18.431 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: pulsar... <<================= check diff failed 5-th time, retry later [Sat May 4 11:11:21 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.60866088.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:21 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b85b28a6-d48a-454b-a3f9-4982d6a1e784 {"id":"b85b28a6-d48a-454b-a3f9-4982d6a1e784","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792278} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41955ae1e8 b85b28a6-d48a-454b-a3f9-4982d6a1e784 /tidb/cdc/default/default/upstream/7364976674316437825 {"id":7364976674316437825,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b85b28a6-d48a-454b-a3f9-4982d6a1e784 {"id":"b85b28a6-d48a-454b-a3f9-4982d6a1e784","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792278} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41955ae1e8 b85b28a6-d48a-454b-a3f9-4982d6a1e784 /tidb/cdc/default/default/upstream/7364976674316437825 {"id":7364976674316437825,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b85b28a6-d48a-454b-a3f9-4982d6a1e784 {"id":"b85b28a6-d48a-454b-a3f9-4982d6a1e784","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792278} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41955ae1e8 b85b28a6-d48a-454b-a3f9-4982d6a1e784 /tidb/cdc/default/default/upstream/7364976674316437825 {"id":7364976674316437825,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0656d2980002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:5383, start at 2024-05-04 11:11:21.25606855 +0800 CST m=+5.157631564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:21.264 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:21.254 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:21.254 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0656d2980002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:5383, start at 2024-05-04 11:11:21.25606855 +0800 CST m=+5.157631564 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:21.264 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:21.254 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:21.254 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0656d3440015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:5467, start at 2024-05-04 11:11:21.340865505 +0800 CST m=+5.190231805 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:21.348 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:21.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:21.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sat May 4 11:11:22 CST 2024] <<<<<< START Pulsar consumer in changefeed_reconstruct case >>>>>> check diff failed 6-th time, retry later ***************** properties ***************** "updateproportion"="0" "recordcount"="50" "readallfields"="true" "threadcount"="4" "mysql.host"="127.0.0.1" "readproportion"="0" "mysql.port"="4000" "operationcount"="0" "workload"="core" "requestdistribution"="uniform" "scanproportion"="0" "mysql.user"="root" "mysql.db"="changefeed_reconstruct" "insertproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 19.057351ms INSERT - Takes(s): 0.0, Count: 46, OPS: 3198.2, Avg(us): 1362, Min(us): 921, Max(us): 4586, 95th(us): 2000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:24 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3414ebbe-0488-4ad8-b372-2988b720c397 {"id":"3414ebbe-0488-4ad8-b372-2988b720c397","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792281} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41957ca5ca 3414ebbe-0488-4ad8-b372-2988b720c397 /tidb/cdc/default/default/upstream/7364976707366518914 {"id":7364976707366518914,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3414ebbe-0488-4ad8-b372-2988b720c397 {"id":"3414ebbe-0488-4ad8-b372-2988b720c397","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792281} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41957ca5ca 3414ebbe-0488-4ad8-b372-2988b720c397 /tidb/cdc/default/default/upstream/7364976707366518914 {"id":7364976707366518914,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3414ebbe-0488-4ad8-b372-2988b720c397 {"id":"3414ebbe-0488-4ad8-b372-2988b720c397","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792281} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41957ca5ca 3414ebbe-0488-4ad8-b372-2988b720c397 /tidb/cdc/default/default/upstream/7364976707366518914 {"id":7364976707366518914,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:11:24 CST 2024] <<<<<< START pulsar cluster in ddl_manager case >>>>>>' [Sat May 4 11:11:24 CST 2024] <<<<<< START pulsar cluster in ddl_manager case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_manager + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_manager + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_manager/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_manager/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_manager/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_manager/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_manager/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_manager/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_manager/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_manager/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_manager/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:11:24 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_manager case >>>>>>' [Sat May 4 11:11:24 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_manager case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_manager/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_manager/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_manager/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ++ date + echo '[Sat May 4 11:11:24 CST 2024] <<<<<< START pulsar cluster in changefeed_pause_resume case >>>>>>' [Sat May 4 11:11:24 CST 2024] <<<<<< START pulsar cluster in changefeed_pause_resume case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_pause_resume + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_pause_resume + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_pause_resume/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_pause_resume/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:11:24 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_pause_resume case >>>>>>' [Sat May 4 11:11:24 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_pause_resume case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... check diff successfully wait process cdc.test exit for 1-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 6.34 secs (588563792 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 2-th time... table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:11:26 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 check diff successfully wait process 6751 exit for 1-th time... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process 6751 exit for 2-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (6751) - No such process wait process 6751 exit for 3-th time... process 6751 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sat May 4 11:11:29 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.70877089.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 public/default public/default ++ date + echo '[Sat May 4 11:11:30 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:11:30 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 11:11:30 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.72217223.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ date + echo '[Sat May 4 11:11:30 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:11:30 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.6556.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json&enable-tidb-extension=true' -c=ddl-manager start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7364976707366518914,"namespace":"default","id":"ddl-manager","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:11:31.720201391+08:00","start_ts":449522510350778374,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522510350778374,"checkpoint_ts":449522510350778374,"checkpoint_time":"2024-05-04 11:11:31.072"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:32 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/9ccbb466-edfd-4c54-8143-64d83874f5cc {UpstreamID:7364976674316437825 Namespace:default ID:9ccbb466-edfd-4c54-8143-64d83874f5cc SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-31594?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:11:22.381206825 +0800 CST StartTs:449522507914674181 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0015d3560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522508072222721} {CheckpointTs:449522509238501378 MinTableBarrierTs:449522509500645378 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522509238501378, checkpointTs: 449522509238501378, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15011bfc-e23d-4392-801e-dbf8b83e3a50 {"id":"15011bfc-e23d-4392-801e-dbf8b83e3a50","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41955ae29b 15011bfc-e23d-4392-801e-dbf8b83e3a50 /tidb/cdc/default/default/changefeed/info/9ccbb466-edfd-4c54-8143-64d83874f5cc {"upstream-id":7364976674316437825,"namespace":"default","changefeed-id":"9ccbb466-edfd-4c54-8143-64d83874f5cc","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-31594?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:11:22.381206825+08:00","start-ts":449522507914674181,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522508072222721} /tidb/cdc/default/default/changefeed/status/9ccbb466-edfd-4c54-8143-64d83874f5cc {"checkpoint-ts":449522509238501378,"min-table-barrier-ts":449522509500645378,"admin-job-type":0} /tidb/cdc/default/default/task/position/15011bfc-e23d-4392-801e-dbf8b83e3a50/9ccbb466-edfd-4c54-8143-64d83874f5cc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976674316437825 {"id":7364976674316437825,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9ccbb466-edfd-4c54-8143-64d83874f5cc {UpstreamID:7364976674316437825 Namespace:default ID:9ccbb466-edfd-4c54-8143-64d83874f5cc SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-31594?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:11:22.381206825 +0800 CST StartTs:449522507914674181 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0015d3560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522508072222721} {CheckpointTs:449522509238501378 MinTableBarrierTs:449522509500645378 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522509238501378, checkpointTs: 449522509238501378, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15011bfc-e23d-4392-801e-dbf8b83e3a50 {"id":"15011bfc-e23d-4392-801e-dbf8b83e3a50","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41955ae29b 15011bfc-e23d-4392-801e-dbf8b83e3a50 /tidb/cdc/default/default/changefeed/info/9ccbb466-edfd-4c54-8143-64d83874f5cc {"upstream-id":7364976674316437825,"namespace":"default","changefeed-id":"9ccbb466-edfd-4c54-8143-64d83874f5cc","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-31594?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:11:22.381206825+08:00","start-ts":449522507914674181,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522508072222721} /tidb/cdc/default/default/changefeed/status/9ccbb466-edfd-4c54-8143-64d83874f5cc {"checkpoint-ts":449522509238501378,"min-table-barrier-ts":449522509500645378,"admin-job-type":0} /tidb/cdc/default/default/task/position/15011bfc-e23d-4392-801e-dbf8b83e3a50/9ccbb466-edfd-4c54-8143-64d83874f5cc {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976674316437825 + grep -q 'failed to get info:' {"id":7364976674316437825,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/9ccbb466-edfd-4c54-8143-64d83874f5cc {UpstreamID:7364976674316437825 Namespace:default ID:9ccbb466-edfd-4c54-8143-64d83874f5cc SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-31594?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:11:22.381206825 +0800 CST StartTs:449522507914674181 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0015d3560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522508072222721} {CheckpointTs:449522509238501378 MinTableBarrierTs:449522509500645378 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522509238501378, checkpointTs: 449522509238501378, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15011bfc-e23d-4392-801e-dbf8b83e3a50 {"id":"15011bfc-e23d-4392-801e-dbf8b83e3a50","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792289} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41955ae29b 15011bfc-e23d-4392-801e-dbf8b83e3a50 /tidb/cdc/default/default/changefeed/info/9ccbb466-edfd-4c54-8143-64d83874f5cc {"upstream-id":7364976674316437825,"namespace":"default","changefeed-id":"9ccbb466-edfd-4c54-8143-64d83874f5cc","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-31594?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:11:22.381206825+08:00","start-ts":449522507914674181,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522508072222721} /tidb/cdc/default/default/changefeed/status/9ccbb466-edfd-4c54-8143-64d83874f5cc {"checkpoint-ts":449522509238501378,"min-table-barrier-ts":449522509500645378,"admin-job-type":0} /tidb/cdc/default/default/task/position/15011bfc-e23d-4392-801e-dbf8b83e3a50/9ccbb466-edfd-4c54-8143-64d83874f5cc {"checkpoint+ grep -q 'etcd info' -ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976674316437825 {"id":7364976674316437825,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id + set +x [Sat May 4 11:11:33 CST 2024] <<<<<< START Pulsar consumer in ddl_manager case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:33 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/964c6f18-9e37-4332-a0bd-70e02c6a4dbd {"id":"964c6f18-9e37-4332-a0bd-70e02c6a4dbd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792291} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419588aee0 964c6f18-9e37-4332-a0bd-70e02c6a4dbd /tidb/cdc/default/default/upstream/7364976735489091122 {"id":7364976735489091122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/964c6f18-9e37-4332-a0bd-70e02c6a4dbd {"id":"964c6f18-9e37-4332-a0bd-70e02c6a4dbd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792291} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419588aee0 964c6f18-9e37-4332-a0bd-70e02c6a4dbd /tidb/cdc/default/default/upstream/7364976735489091122 {"id":7364976735489091122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/964c6f18-9e37-4332-a0bd-70e02c6a4dbd {"id":"964c6f18-9e37-4332-a0bd-70e02c6a4dbd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792291} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419588aee0 964c6f18-9e37-4332-a0bd-70e02c6a4dbd /tidb/cdc/default/default/upstream/7364976735489091122 {"id":7364976735489091122,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release "id": "15011bfc-e23d-4392-801e-dbf8b83e3a50", "cluster-id": "default" run task successfully [Sat May 4 11:11:34 CST 2024] <<<<<< START Pulsar consumer in changefeed_pause_resume case >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) capture_id: 15011bfc-e23d-4392-801e-dbf8b83e3a50 check_processor_table_count http://127.0.0.1:2379 9ccbb466-edfd-4c54-8143-64d83874f5cc 15011bfc-e23d-4392-801e-dbf8b83e3a50 1 run task successfully check_processor_table_count http://127.0.0.1:2379 9ccbb466-edfd-4c54-8143-64d83874f5cc 15011bfc-e23d-4392-801e-dbf8b83e3a50 0 table count 1 does equal to expected count 0 run task failed 1-th time, retry later table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:11:35 CST 2024] <<<<<< run test case sink_retry success! >>>>>> table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later wait process 6091 exit for 1-th time... wait process 6091 exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 6091 exit for 3-th time... check_processor_table_count http://127.0.0.1:2379 9ccbb466-edfd-4c54-8143-64d83874f5cc 15011bfc-e23d-4392-801e-dbf8b83e3a50 0 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (6091) - No such process wait process 6091 exit for 4-th time... process 6091 already exit [Sat May 4 11:11:38 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.66836685.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 run task successfully ***************** properties ***************** "mysql.user"="root" "operationcount"="0" "insertproportion"="0" "requestdistribution"="uniform" "dotransactions"="false" "recordcount"="50" "readallfields"="true" "readproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "threadcount"="4" "mysql.db"="changefeed_reconstruct" "mysql.port"="4000" "workload"="core" ********************************************** Run finished, takes 20.321766ms INSERT - Takes(s): 0.0, Count: 48, OPS: 2964.9, Avg(us): 1599, Min(us): 1037, Max(us): 4229, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 11.52 secs (323832117 bytes/sec) [Pipeline] { [Pipeline] cache table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table changefeed_pause_resume.t3 exists table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff successfully wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658085c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89, pid:5032, start at 2024-05-04 11:11:41.084714784 +0800 CST m=+5.209020568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:41.091 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:41.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:41.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658085c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89, pid:5032, start at 2024-05-04 11:11:41.084714784 +0800 CST m=+5.209020568 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:41.091 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:41.079 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:41.079 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065809d40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-blcq-5gl89, pid:5109, start at 2024-05-04 11:11:41.176565714 +0800 CST m=+5.244825979 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:41.183 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:41.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:41.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:11:44 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:44 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364976707366518914 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:11:31.720201391 +0800 CST StartTs:449522510350778374 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00154ac60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522510508326919} {CheckpointTs:449522510914650137 MinTableBarrierTs:449522510914650137 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa {"id":"4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41957ca877 4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364976707366518914,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:11:31.720201391+08:00","start-ts":449522510350778374,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522510508326919} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522510914650137,"min-table-barrier-ts":449522510914650137,"admin-job-type":0} /tidb/cdc/default/default/task/position/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976707366518914 {"id":7364976707366518914,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364976707366518914 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:11:31.720201391 +0800 CST StartTs:449522510350778374 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00154ac60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522510508326919} {CheckpointTs:449522510914650137 MinTableBarrierTs:449522510914650137 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa {"id":"4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41957ca877 4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364976707366518914,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:11:31.720201391+08:00","start-ts":449522510350778374,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522510508326919} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522510914650137,"min-table-barrier-ts":449522510914650137,"admin-job-type":0} /tidb/cdc/default/default/task/position/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976707366518914 {"id":7364976707366518914,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364976707366518914 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:11:31.720201391 +0800 CST StartTs:449522510350778374 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00154ac60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522510508326919} {CheckpointTs:449522510914650137 MinTableBarrierTs:449522510914650137 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa {"id":"4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41957ca877 4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364976707366518914,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:11:31.720201391+08:00","start-ts":449522510350778374,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522510508326919} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522510914650137,"min-table-barrier-ts":449522510914650137,"admin-job-type":0} /tidb/cdc/default/default/task/position/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976707366518914 {"id":7364976707366518914,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:11:44 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.67476749.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.6450.out cli tso query --pd=http://127.0.0.1:2379 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 6.52 secs (572406557 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 2-th time, retry later + set +x + tso='449522514403000322 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522514403000322 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later [Sat May 4 11:11:49 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.65056507.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:49 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364976707366518914 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:11:31.720201391 +0800 CST StartTs:449522510350778374 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00154ac60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522510508326919} {CheckpointTs:449522510914650137 MinTableBarrierTs:449522510914650137 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa {"id":"4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41957ca877 4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364976707366518914,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:11:31.720201391+08:00","start-ts":449522510350778374,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522510508326919} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522510927495204,"min-table-barrier-ts":449522510927495204,"admin-job-type":0} /tidb/cdc/default/default/task/position/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976707366518914 {"id":7364976707366518914,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364976707366518914 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:11:31.720201391 +0800 CST StartTs:449522510350778374 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00154ac60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522510508326919} {CheckpointTs:449522510914650137 MinTableBarrierTs:449522510914650137 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa {"id":"4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41957ca877 4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364976707366518914,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:11:31.720201391+08:00","start-ts":449522510350778374,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522510508326919} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522510927495204,"min-table-barrier-ts":449522510927495204,"admin-job-type":0} /tidb/cdc/default/default/task/position/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976707366518914 {"id":7364976707366518914,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7364976707366518914 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:11:31.720201391 +0800 CST StartTs:449522510350778374 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00154ac60 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522510508326919} {CheckpointTs:449522510914650137 MinTableBarrierTs:449522510914650137 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa {"id":"4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41957ca877 4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7364976707366518914,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-12971?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:11:31.720201391+08:00","start-ts":449522510350778374,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522510508326919} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449522510927495204,"min-table-barrier-ts":449522510927495204,"admin-job-type":0} /tidb/cdc/default/default/task/position/4a0ca8e2-2050-4f4a-83da-d1e1b6bdd2aa/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364976707366518914 {"id":7364976707366518914,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 2-th check, retry later check diff failed 4-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:11:52 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a0a05a20-b146-483c-90a3-d7db28cd0834 {"id":"a0a05a20-b146-483c-90a3-d7db28cd0834","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792309} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4195d9d603 a0a05a20-b146-483c-90a3-d7db28cd0834 /tidb/cdc/default/default/upstream/7364976816472258861 {"id":7364976816472258861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a0a05a20-b146-483c-90a3-d7db28cd0834 {"id":"a0a05a20-b146-483c-90a3-d7db28cd0834","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792309} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4195d9d603 a0a05a20-b146-483c-90a3-d7db28cd0834 /tidb/cdc/default/default/upstream/7364976816472258861 {"id":7364976816472258861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a0a05a20-b146-483c-90a3-d7db28cd0834 {"id":"a0a05a20-b146-483c-90a3-d7db28cd0834","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792309} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4195d9d603 a0a05a20-b146-483c-90a3-d7db28cd0834 /tidb/cdc/default/default/upstream/7364976816472258861 {"id":7364976816472258861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:11:52 CST 2024] <<<<<< START pulsar cluster in split_region case >>>>>>' [Sat May 4 11:11:52 CST 2024] <<<<<< START pulsar cluster in split_region case >>>>>> + workdir=/tmp/tidb_cdc_test/split_region + cluster_type=oauth + cd /tmp/tidb_cdc_test/split_region + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/split_region/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/split_region/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/split_region/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/split_region/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/split_region/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/split_region/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/split_region/pulsar_standalone.conf + pulsar_port=6650 + '[' oauth == mtls ']' + '[' oauth == oauth ']' + echo 'Waiting for oauth2 server to be ready...' Waiting for oauth2 server to be ready... + i=0 + nc -z localhost 9096 + oauth2-server + gen_oauth_config + echo ' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ date + echo '[Sat May 4 11:11:52 CST 2024] <<<<<< START pulsar cluster in oauth mode in split_region case >>>>>>' [Sat May 4 11:11:52 CST 2024] <<<<<< START pulsar cluster in oauth mode in split_region case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/split_region/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/split_region/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/split_region/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658b00c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:11542, start at 2024-05-04 11:11:51.816341045 +0800 CST m=+5.561505937 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:51.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:51.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:51.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658b00c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:11542, start at 2024-05-04 11:11:51.816341045 +0800 CST m=+5.561505937 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:51.826 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:51.811 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:51.811 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0658aff40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-b63n-27ks8, pid:11629, start at 2024-05-04 11:11:51.843028463 +0800 CST m=+5.536293834 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:13:51.853 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:11:51.856 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:01:51.856 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 3-th check, retry later check diff failed 5-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.12964.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 4-th check, retry later check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + set +x + tso='449522516823113729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522516823113729 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:11:57 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1301413016.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 5-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 6-th check, retry later check diff failed 1-th time, retry later public/default ++ date + echo '[Sat May 4 11:12:00 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:12:00 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.6981.out cli changefeed create --start-ts=449522514403000322 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-split-region-test-11416?protocol=canal-json&enable-tidb-extension=true' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:00 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/567da1a1-d6f7-4293-b80c-42ec41938910 {"id":"567da1a1-d6f7-4293-b80c-42ec41938910","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960840d1 567da1a1-d6f7-4293-b80c-42ec41938910 /tidb/cdc/default/default/upstream/7364976863651284494 {"id":7364976863651284494,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/567da1a1-d6f7-4293-b80c-42ec41938910 {"id":"567da1a1-d6f7-4293-b80c-42ec41938910","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960840d1 567da1a1-d6f7-4293-b80c-42ec41938910 /tidb/cdc/default/default/upstream/7364976863651284494 {"id":7364976863651284494,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/567da1a1-d6f7-4293-b80c-42ec41938910 {"id":"567da1a1-d6f7-4293-b80c-42ec41938910","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792317} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41960840d1 567da1a1-d6f7-4293-b80c-42ec41938910 /tidb/cdc/default/default/upstream/7364976863651284494 {"id":7364976863651284494,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:12:00 CST 2024] <<<<<< START pulsar cluster in partition_table case >>>>>>' [Sat May 4 11:12:00 CST 2024] <<<<<< START pulsar cluster in partition_table case >>>>>> + workdir=/tmp/tidb_cdc_test/partition_table + cluster_type=normal + cd /tmp/tidb_cdc_test/partition_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/partition_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/partition_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/partition_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/partition_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/partition_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/partition_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/partition_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/partition_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/partition_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/partition_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/partition_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/partition_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:00 CST 2024] <<<<<< START pulsar cluster in normal mode in partition_table case >>>>>>' [Sat May 4 11:12:00 CST 2024] <<<<<< START pulsar cluster in normal mode in partition_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/partition_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/partition_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/partition_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 7-th check, retry later Create changefeed successfully! ID: c268b98a-f169-4eb9-ac62-0b4d218d0a99 Info: {"upstream_id":7364976816472258861,"namespace":"default","id":"c268b98a-f169-4eb9-ac62-0b4d218d0a99","sink_uri":"pulsar://127.0.0.1:6650/ticdc-split-region-test-11416?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:12:01.453498523+08:00","start_ts":449522514403000322,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"oauth2":{"oauth2-issuer-url":"http://localhost:9096","oauth2-audience":"cdc-api-uri","oauth2-private-key":"/tmp/tidb_cdc_test/split_region/credential.json","oauth2-client-id":"1234"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522514403000322,"checkpoint_ts":449522514403000322,"checkpoint_time":"2024-05-04 11:11:46.530"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 2-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + set +x [Sat May 4 11:12:02 CST 2024] <<<<<< START Pulsar consumer in split_region case >>>>>> table split_region.test1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 8-th check, retry later check diff failed 3-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 14.59 secs (255680235 bytes/sec) [Pipeline] { [Pipeline] cache + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06596e4c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:4709, start at 2024-05-04 11:12:04.0288059 +0800 CST m=+5.297569266 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:04.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:04.038 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:04.038 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 9-th check, retry later check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06596e4c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:4709, start at 2024-05-04 11:12:04.0288059 +0800 CST m=+5.297569266 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:04.038 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:04.038 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:04.038 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06596fc00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:4793, start at 2024-05-04 11:12:04.10807729 +0800 CST m=+5.323322000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:04.116 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:04.080 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:04.080 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } public/default ++ date + echo '[Sat May 4 11:12:07 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:12:07 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.13487.out cli changefeed create --start-ts=449522516823113729 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-partition-table-test-17822?protocol=canal-json&enable-tidb-extension=true' table split_region.test2 exists check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 10-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 3bcf0eb1-9c46-4ccd-8646-0ae6f43aa23f Info: {"upstream_id":7364976863651284494,"namespace":"default","id":"3bcf0eb1-9c46-4ccd-8646-0ae6f43aa23f","sink_uri":"pulsar://127.0.0.1:6650/ticdc-partition-table-test-17822?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:12:08.392018498+08:00","start_ts":449522516823113729,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522516823113729,"checkpoint_ts":449522516823113729,"checkpoint_time":"2024-05-04 11:11:55.762"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully check diff failed 5-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6188.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 11-th check, retry later + set +x [Sat May 4 11:12:09 CST 2024] <<<<<< START Pulsar consumer in partition_table case >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449522520395087873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522520395087873 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "mysql.host"="127.0.0.1" "workload"="core" "updateproportion"="0" "insertproportion"="0" "mysql.port"="4000" "readallfields"="true" "dotransactions"="false" "threadcount"="4" "mysql.user"="root" "readproportion"="0" "mysql.db"="changefeed_error" "scanproportion"="0" "recordcount"="20" "operationcount"="0" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.427127ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3656.1, Avg(us): 1779, Min(us): 1063, Max(us): 3955, 95th(us): 4000, 99th(us): 4000 [Sat May 4 11:12:11 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62506252.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully table ddl_manager.finish_mark not exists for 12-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 13-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:14 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 {"id":"1dac1a66-de27-40aa-b27d-fb779fbeb8b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792331} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f38ee 1dac1a66-de27-40aa-b27d-fb779fbeb8b3 /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 {"id":"1dac1a66-de27-40aa-b27d-fb779fbeb8b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792331} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f38ee 1dac1a66-de27-40aa-b27d-fb779fbeb8b3 /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 {"id":"1dac1a66-de27-40aa-b27d-fb779fbeb8b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792331} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f38ee 1dac1a66-de27-40aa-b27d-fb779fbeb8b3 /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:12:14 CST 2024] <<<<<< START pulsar cluster in changefeed_error case >>>>>>' [Sat May 4 11:12:14 CST 2024] <<<<<< START pulsar cluster in changefeed_error case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_error + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_error + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_error/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_error/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_error/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_error/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_error/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_error/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_error/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_error/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_error/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:14 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_error case >>>>>>' [Sat May 4 11:12:14 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_error case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_error/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_error/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_error/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 14-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0659fe800005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:8619, start at 2024-05-04 11:12:13.221377947 +0800 CST m=+5.268541080 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:13.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:13.216 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:13.216 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0659fe800005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:8619, start at 2024-05-04 11:12:13.221377947 +0800 CST m=+5.268541080 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:13.229 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:13.216 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:13.216 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0659fe600015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:8707, start at 2024-05-04 11:12:13.249413285 +0800 CST m=+5.243543391 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:13.256 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:13.258 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:13.258 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 15-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.10040.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 16-th check, retry later + set +x + tso='449522522840367105 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522522840367105 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "readproportion"="0" "readallfields"="true" "insertproportion"="0" "mysql.db"="multi_capture_1" "mysql.port"="4000" "workload"="core" "dotransactions"="false" "scanproportion"="0" "mysql.user"="root" "updateproportion"="0" "threadcount"="2" "recordcount"="10" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "operationcount"="0" ********************************************** Run finished, takes 8.231411ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2272.5, Avg(us): 1565, Min(us): 935, Max(us): 3764, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "scanproportion"="0" "mysql.user"="root" "mysql.port"="4000" "updateproportion"="0" "readproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "requestdistribution"="uniform" "operationcount"="0" "readallfields"="true" "recordcount"="10" "mysql.db"="multi_capture_2" "insertproportion"="0" "threadcount"="2" "workload"="core" ********************************************** Run finished, takes 8.884769ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2036.5, Avg(us): 1703, Min(us): 977, Max(us): 3921, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "mysql.user"="root" "mysql.port"="4000" "readproportion"="0" "insertproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "operationcount"="0" "requestdistribution"="uniform" "recordcount"="10" "dotransactions"="false" "threadcount"="2" "scanproportion"="0" "updateproportion"="0" "mysql.db"="multi_capture_3" "readallfields"="true" ********************************************** Run finished, takes 9.507475ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1839.3, Avg(us): 1808, Min(us): 1129, Max(us): 4005, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "insertproportion"="0" "requestdistribution"="uniform" "updateproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "readallfields"="true" "recordcount"="10" "workload"="core" "readproportion"="0" "mysql.user"="root" "mysql.db"="multi_capture_4" "operationcount"="0" "dotransactions"="false" "scanproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 8.543833ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2097.3, Avg(us): 1627, Min(us): 950, Max(us): 3669, 95th(us): 4000, 99th(us): 4000 [Sat May 4 11:12:21 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1016410166.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 17-th check, retry later public/default ++ date + echo '[Sat May 4 11:12:22 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:12:22 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6717.out cli changefeed create --start-ts=449522520395087873 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7364976913536362088,"namespace":"default","id":"changefeed-error","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:12:23.309025194+08:00","start_ts":449522520395087873,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522520395087873,"checkpoint_ts":449522520395087873,"checkpoint_time":"2024-05-04 11:12:09.388"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:24 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1a4cf96-5cf1-470e-af8b-6551c7f20d40 {"id":"b1a4cf96-5cf1-470e-af8b-6551c7f20d40","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792341} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419653463b b1a4cf96-5cf1-470e-af8b-6551c7f20d40 /tidb/cdc/default/default/upstream/7364976952550935526 {"id":7364976952550935526,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1a4cf96-5cf1-470e-af8b-6551c7f20d40 {"id":"b1a4cf96-5cf1-470e-af8b-6551c7f20d40","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792341} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419653463b b1a4cf96-5cf1-470e-af8b-6551c7f20d40 /tidb/cdc/default/default/upstream/7364976952550935526 {"id":7364976952550935526,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1a4cf96-5cf1-470e-af8b-6551c7f20d40 {"id":"b1a4cf96-5cf1-470e-af8b-6551c7f20d40","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792341} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419653463b b1a4cf96-5cf1-470e-af8b-6551c7f20d40 /tidb/cdc/default/default/upstream/7364976952550935526 {"id":7364976952550935526,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:12:24 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1021710219.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later table partition_table.t2 not exists for 2-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc) 3730338816 bytes in 18.42 secs (202530743 bytes/sec) [Pipeline] { [Pipeline] sh + set +x [Sat May 4 11:12:24 CST 2024] <<<<<< START Pulsar consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449522520395087873, "checkpoint_time": "2024-05-04 11:12:09.388", "error": { "time": "2024-05-04T11:12:23.381662717+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522520395087573 is earlier than or equal to GC safepoint at 449522520395087873" } }' + echo '{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449522520395087873, "checkpoint_time": "2024-05-04 11:12:09.388", "error": { "time": "2024-05-04T11:12:23.381662717+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522520395087573 is earlier than or equal to GC safepoint at 449522520395087873" } }' { "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449522520395087873, "checkpoint_time": "2024-05-04 11:12:09.388", "error": { "time": "2024-05-04T11:12:23.381662717+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522520395087573 is earlier than or equal to GC safepoint at 449522520395087873" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449522520395087873, '"checkpoint_time":' '"2024-05-04' '11:12:09.388",' '"error":' '{' '"time":' '"2024-05-04T11:12:23.381662717+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449522520395087573 is earlier than or equal to GC safepoint at '449522520395087873"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449522520395087873, '"checkpoint_time":' '"2024-05-04' '11:12:09.388",' '"error":' '{' '"time":' '"2024-05-04T11:12:23.381662717+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449522520395087573 is earlier than or equal to GC safepoint at '449522520395087873"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522520395087573 is earlier than or equal to GC safepoint at 449522520395087873' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449522520395087573 is earlier than or equal to GC safepoint at 449522520395087873 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6847.out cli changefeed resume -c changefeed-error [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=2b039f07-ae8c-41ec-8543-7eb199410797 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-xns4r GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-xns4r GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: pulsar... <<================= [Pipeline] sh [Pipeline] sh check diff successfully wait process cdc.test exit for 1-th time... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff successfully table ddl_manager.finish_mark not exists for 19-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=588af93e-898c-484c-b642-f3c63b70c156 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-h6bt9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-h6bt9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: pulsar... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=9a86294c-4a49-4ae6-9b30-cd44d160bcc6 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-79hx9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-79hx9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: pulsar... <<================= [Pipeline] sh wait process cdc.test exit for 2-th time... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G14 [Pipeline] sh wait process cdc.test exit for 3-th time... Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=f6b90fdb-5af9-4365-832a-51843bfd0737 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-cnnh9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9 pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-cnnh9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=15d2f8b8-c174-4819-8ac1-d14558425288 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-rl3x1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-rl3x1 pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later + set +x table changefeed_error.usertable not exists for 1-th check, retry later [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=80fe2802-c6ad-4b32-bf30-f5b922f21832 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-ptr6b GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-ptr6b pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: pulsar... <<================= cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:12:27 CST 2024] <<<<<< run test case split_region success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=063f093e-0841-4108-8036-8c86627345a8 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-4f64-16spm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-4f64p GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-4f64p pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-4f64-16spm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-4f64-16spm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:27 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> table ddl_manager.finish_mark not exists for 20-th check, retry later [Pipeline] // node + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=f7659fc8-dac9-43af-bc8f-0099c4e37028 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-dhfzd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-dhfzd pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:28 GMT < Content-Length: 1273 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1a4cf96-5cf1-470e-af8b-6551c7f20d40 {"id":"b1a4cf96-5cf1-470e-af8b-6551c7f20d40","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792341} /tidb/cdc/default/__cdc_meta__/capture/d68548b1-32b2-4db9-8aeb-acba816cf149 {"id":"d68548b1-32b2-4db9-8aeb-acba816cf149","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792344} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419653463b b1a4cf96-5cf1-470e-af8b-6551c7f20d40 /tidb/cdc/default/__cdc_meta__/owner/22318f419653466c d68548b1-32b2-4db9-8aeb-acba816cf149 /tidb/cdc/default/default/upstream/7364976952550935526 {"id":7364976952550935526,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1a4cf96-5cf1-470e-af8b-6551c7f20d40 {"id":"b1a4cf96-5cf1-470e-af8b-6551c7f20d40","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792341} /tidb/cdc/default/__cdc_meta__/capture/d68548b1-32b2-4db9-8aeb-acba816cf149 {"id":"d68548b1-32b2-4db9-8aeb-acba816cf149","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792344} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419653463b b1a4cf96-5cf1-470e-af8b-6551c7f20d40 /tidb/cdc/default/__cdc_meta__/owner/22318f419653466c d68548b1-32b2-4db9-8aeb-acba816cf149 /tidb/cdc/default/default/upstream/7364976952550935526 {"id":7364976952550935526,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b1a4cf96-5cf1-470e-af8b-6551c7f20d40 {"id":"b1a4cf96-5cf1-470e-af8b-6551c7f20d40","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792341} /tidb/cdc/default/__cdc_meta__/capture/d68548b1-32b2-4db9-8aeb-acba816cf149 {"id":"d68548b1-32b2-4db9-8aeb-acba816cf149","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792344} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419653463b b1a4cf96-5cf1-470e-af8b-6551c7f20d40 /tidb/cdc/default/__cdc_meta__/owner/22318f419653466c d68548b1-32b2-4db9-8aeb-acba816cf149 /tidb/cdc/default/default/upstream/7364976952550935526 {"id":7364976952550935526,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:12:28 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1026810270.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=408ace03-209f-4e1f-bd0d-cbe7ba845d1a JENKINS_NODE_COOKIE=7b37e4ed-7b5a-4046-bfe0-ef51d96e128b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786590407547686913","prowjobid":"408ace03-209f-4e1f-bd0d-cbe7ba845d1a","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/b297eae37cdb06fa18bc5388b04d2564a81a0b87","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1498 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786590407547686913 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1498/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-t0rdr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb pingcap_tiflow_pull_cdc_integration_pulsar_test_1498-t0rdr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1498 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:28 CST 2024] <<<<<< skip test case clustered_index for pulsar! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table partition_table.finish_mark not exists for 2-th check, retry later table changefeed_error.usertable exists check diff failed 1-th time, retry later The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 21-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] } table partition_table.finish_mark not exists for 3-th check, retry later check diff failed 1-th time, retry later check diff successfully start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ***************** properties ***************** "mysql.port"="4000" "scanproportion"="0" "mysql.host"="127.0.0.1" "updateproportion"="0" "dotransactions"="false" "insertproportion"="0" "mysql.user"="root" "mysql.db"="changefeed_error" "threadcount"="4" "requestdistribution"="uniform" "operationcount"="0" "workload"="core" "recordcount"="20" "readallfields"="true" "readproportion"="0" ********************************************** Run finished, takes 5.935284ms INSERT - Takes(s): 0.0, Count: 20, OPS: 5254.2, Avg(us): 1042, Min(us): 599, Max(us): 2420, 95th(us): 3000, 99th(us): 3000 check diff successfully {"id":"1dac1a66-de27-40aa-b27d-fb779fbeb8b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792331} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 + [[ /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 22-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:31 GMT < Content-Length: 1753 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ebc863e-df48-462b-b07b-c45e7487d307 {"id":"2ebc863e-df48-462b-b07b-c45e7487d307","address":"127.0.0.1:8303","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792348} /tidb/cdc/default/__cdc_meta__/capture/b1a4cf96-5cf1-470e-af8b-6551c7f20d40 {"id":"b1a4cf96-5cf1-470e-af8b-6551c7f20d40","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792341} /tidb/cdc/default/__cdc_meta__/capture/d68548b1-32b2-4db9-8aeb-acba816cf149 {"id":"d68548b1-32b2-4db9-8aeb-acba816cf149","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792344} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419653463b b1a4cf96-5cf1-470e-af8b-6551c7f20d40 /tidb/cdc/default/__cdc_meta__/owner/22318f419653466c d68548b1-32b2-4db9-8aeb-acba816cf149 /tidb/cdc/default/__cdc_meta__/owner/22318f419653469a 2ebc863e-df48-462b-b07b-c45e7487d307 /tidb/cdc/default/default/upstream/7364976952550935526 {"id":7364976952550935526,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ebc863e-df48-462b-b07b-c45e7487d307 {"id":"2ebc863e-df48-462b-b07b-c45e7487d307","address":"127.0.0.1:8303","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792348} /tidb/cdc/default/__cdc_meta__/capture/b1a4cf96-5cf1-470e-af8b-6551c7f20d40 {"id":"b1a4cf96-5cf1-470e-af8b-6551c7f20d40","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792341} /tidb/cdc/default/__cdc_meta__/capture/d68548b1-32b2-4db9-8aeb-acba816cf149 {"id":"d68548b1-32b2-4db9-8aeb-acba816cf149","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792344} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419653463b b1a4cf96-5cf1-470e-af8b-6551c7f20d40 /tidb/cdc/default/__cdc_meta__/owner/22318f419653466c d68548b1-32b2-4db9-8aeb-acba816cf149 /tidb/cdc/default/__cdc_meta__/owner/22318f419653469a 2ebc863e-df48-462b-b07b-c45e7487d307 /tidb/cdc/default/default/upstream/7364976952550935526 {"id":7364976952550935526,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ebc863e-df48-462b-b07b-c45e7487d307 {"id":"2ebc863e-df48-462b-b07b-c45e7487d307","address":"127.0.0.1:8303","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792348} /tidb/cdc/default/__cdc_meta__/capture/b1a4cf96-5cf1-470e-af8b-6551c7f20d40 {"id":"b1a4cf96-5cf1-470e-af8b-6551c7f20d40","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792341} /tidb/cdc/default/__cdc_meta__/capture/d68548b1-32b2-4db9-8aeb-acba816cf149 {"id":"d68548b1-32b2-4db9-8aeb-acba816cf149","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792344} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419653463b b1a4cf96-5cf1-470e-af8b-6551c7f20d40 /tidb/cdc/default/__cdc_meta__/owner/22318f419653466c d68548b1-32b2-4db9-8aeb-acba816cf149 /tidb/cdc/default/__cdc_meta__/owner/22318f419653469a 2ebc863e-df48-462b-b07b-c45e7487d307 /tidb/cdc/default/default/upstream/7364976952550935526 {"id":7364976952550935526,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:12:31 CST 2024] <<<<<< START pulsar cluster in multi_capture case >>>>>>' [Sat May 4 11:12:31 CST 2024] <<<<<< START pulsar cluster in multi_capture case >>>>>> + workdir=/tmp/tidb_cdc_test/multi_capture + cluster_type=normal + cd /tmp/tidb_cdc_test/multi_capture + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/multi_capture/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/multi_capture/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/multi_capture/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/multi_capture/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/multi_capture/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/multi_capture/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/multi_capture/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/multi_capture/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/multi_capture/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/multi_capture/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/multi_capture/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/multi_capture/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:31 CST 2024] <<<<<< START pulsar cluster in normal mode in multi_capture case >>>>>>' [Sat May 4 11:12:31 CST 2024] <<<<<< START pulsar cluster in normal mode in multi_capture case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/multi_capture/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/multi_capture/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/multi_capture/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:31 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Verifying upstream PD is started... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table partition_table.finish_mark not exists for 4-th check, retry later Verifying upstream PD is started... check diff failed 2-th time, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 + [[ /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table partition_table.finish_mark not exists for 5-th check, retry later check diff failed 3-th time, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: pulsar... <<================= kafka downstream isn't support syncpoint record [Sat May 4 11:12:34 CST 2024] <<<<<< run test case syncpoint success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 24-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... table partition_table.finish_mark not exists for 6-th check, retry later Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 + [[ /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1dac1a66-de27-40aa-b27d-fb779fbeb8b3 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 25-th check, retry later Verifying downstream PD is started... Starting Upstream TiDB... public/default ++ date + echo '[Sat May 4 11:12:38 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:12:38 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.10728.out cli changefeed create --start-ts=449522522840367105 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-multi-capture-test-17532?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:38 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: a437531a-21f3-4ee4-a193-f0de4e595bd7 Info: {"upstream_id":7364976952550935526,"namespace":"default","id":"a437531a-21f3-4ee4-a193-f0de4e595bd7","sink_uri":"pulsar://127.0.0.1:6650/ticdc-multi-capture-test-17532?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:12:39.112722357+08:00","start_ts":449522522840367105,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522522840367105,"checkpoint_ts":449522522840367105,"checkpoint_time":"2024-05-04 11:12:18.716"} PASS Verifying downstream PD is started... table partition_table.finish_mark not exists for 7-th check, retry later coverage: 2.4% of statements in github.com/pingcap/tiflow/... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 26-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Sat May 4 11:12:40 CST 2024] <<<<<< START Pulsar consumer in multi_capture case >>>>>> table multi_capture_1.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark exists check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:41 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> Starting Upstream TiDB... wait process cdc.test exit for 2-th time... [Pipeline] // podTemplate ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bb08c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9, pid:1365, start at 2024-05-04 11:12:41.036667929 +0800 CST m=+5.162716181 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:41.042 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:41.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:41.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 27-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m wait process cdc.test exit for 3-th time... [Pipeline] // cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:12:42 CST 2024] <<<<<< run test case partition_table success! >>>>>> table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bc9740012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:1426, start at 2024-05-04 11:12:42.619595056 +0800 CST m=+5.240973552 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:42.626 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:42.589 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:42.589 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 11:12:43 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.71167118.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bb08c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9, pid:1365, start at 2024-05-04 11:12:41.036667929 +0800 CST m=+5.162716181 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:41.042 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:41.045 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:41.045 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bbb6c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9, pid:1448, start at 2024-05-04 11:12:41.728474237 +0800 CST m=+5.787814635 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:41.735 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:41.691 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:41.691 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 28-th check, retry later [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bdb3c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr, pid:1387, start at 2024-05-04 11:12:43.775681741 +0800 CST m=+5.314075201 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:43.783 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:43.777 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:43.777 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bd5dc001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2, pid:1305, start at 2024-05-04 11:12:43.429930761 +0800 CST m=+5.170767991 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:43.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:43.433 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:43.433 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bd96c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt, pid:1333, start at 2024-05-04 11:12:43.648246788 +0800 CST m=+5.250299157 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:43.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:43.660 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:43.660 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bc0dc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb, pid:1429, start at 2024-05-04 11:12:42.039121942 +0800 CST m=+5.335837437 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:42.046 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:42.039 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:42.039 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bc0dc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb, pid:1429, start at 2024-05-04 11:12:42.039121942 +0800 CST m=+5.335837437 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:42.046 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:42.039 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:42.039 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bc1940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-t0rd-1crdb, pid:1498, start at 2024-05-04 11:12:42.119074346 +0800 CST m=+5.348130845 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:42.125 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:42.085 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:42.085 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // dir [Pipeline] } find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:44 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> [Pipeline] // stage [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bc9740012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:1426, start at 2024-05-04 11:12:42.619595056 +0800 CST m=+5.240973552 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:42.626 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:42.589 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:42.589 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bcb24000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:1507, start at 2024-05-04 11:12:42.709430097 +0800 CST m=+5.277784544 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:42.718 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:42.697 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:42.697 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // container [Pipeline] } table ddl_manager.finish_mark not exists for 29-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bdb3c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr, pid:1387, start at 2024-05-04 11:12:43.775681741 +0800 CST m=+5.314075201 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:43.783 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:43.777 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:43.777 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bdd2c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr, pid:1462, start at 2024-05-04 11:12:43.865935955 +0800 CST m=+5.346092147 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:43.873 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:43.851 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:43.851 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bd5dc001d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2, pid:1305, start at 2024-05-04 11:12:43.429930761 +0800 CST m=+5.170767991 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:43.436 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:43.433 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:43.433 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bd750000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2, pid:1388, start at 2024-05-04 11:12:43.490267139 +0800 CST m=+5.176614728 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:43.497 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:43.476 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:43.476 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc4b787c-57ce-46c0-8fcf-fef4215f8fa1 {"id":"fc4b787c-57ce-46c0-8fcf-fef4215f8fa1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792363} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3a13 fc4b787c-57ce-46c0-8fcf-fef4215f8fa1 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7364976913536362088,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:12:23.309025194+08:00","start-ts":449522520395087873,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T11:12:44.182106057+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522529504591877} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449522526004707331,"min-table-barrier-ts":449522526004707334,"admin-job-type":1} /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc4b787c-57ce-46c0-8fcf-fef4215f8fa1 {"id":"fc4b787c-57ce-46c0-8fcf-fef4215f8fa1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792363} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3a13 fc4b787c-57ce-46c0-8fcf-fef4215f8fa1 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7364976913536362088,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:12:23.309025194+08:00","start-ts":449522520395087873,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T11:12:44.182106057+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522529504591877} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449522526004707331,"min-table-barrier-ts":449522526004707334,"admin-job-type":1} /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fc4b787c-57ce-46c0-8fcf-fef4215f8fa1 {"id":"fc4b787c-57ce-46c0-8fcf-fef4215f8fa1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792363} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3a13 fc4b787c-57ce-46c0-8fcf-fef4215f8fa1 /tidb/cdc/default/default/changefeed/info/changefeed-error [Pipeline] } {"upstream-id":7364976913536362088,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:12:23.309025194+08:00","start-ts":449522520395087873,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T11:12:44.182106057+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522529504591877} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449522526004707331,"min-table-barrier-ts":449522526004707334,"admin-job-type":1} /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s ++ date + echo '[Sat May 4 11:12:46 CST 2024] <<<<<< START pulsar cluster in changefeed_finish case >>>>>>' [Sat May 4 11:12:46 CST 2024] <<<<<< START pulsar cluster in changefeed_finish case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_finish + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_finish + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_finish/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_finish/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_finish/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_finish/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_finish/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_finish/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_finish/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_finish/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_finish/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:46 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_finish case >>>>>>' [Sat May 4 11:12:46 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_finish case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_finish/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_finish/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_finish/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Pipeline] // withEnv VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bd96c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt, pid:1333, start at 2024-05-04 11:12:43.648246788 +0800 CST m=+5.250299157 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:43.655 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:43.660 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:43.660 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065bdd380014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt, pid:1402, start at 2024-05-04 11:12:43.874722776 +0800 CST m=+5.411714594 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:43.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:43.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:43.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } + info='{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449522526004707331, "checkpoint_time": "2024-05-04 11:12:30.787", "error": { "time": "2024-05-04T11:12:44.182106057+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449522526004707331, "checkpoint_time": "2024-05-04 11:12:30.787", "error": { "time": "2024-05-04T11:12:44.182106057+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449522526004707331, "checkpoint_time": "2024-05-04 11:12:30.787", "error": { "time": "2024-05-04T11:12:44.182106057+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449522526004707331, '"checkpoint_time":' '"2024-05-04' '11:12:30.787",' '"error":' '{' '"time":' '"2024-05-04T11:12:44.182106057+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449522526004707331, '"checkpoint_time":' '"2024-05-04' '11:12:30.787",' '"error":' '{' '"time":' '"2024-05-04T11:12:44.182106057+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7215.out cli changefeed remove -c changefeed-error [Pipeline] // podTemplate table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // podTemplate check diff failed 2-th time, retry later [Pipeline] } [Pipeline] // stage ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Sat May 4 11:12:47 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.28392841.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2892.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] // withEnv [Pipeline] } Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449522526004707331 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json&enable-tidb-extension=true PASS [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 30-th check, retry later coverage: 2.0% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:48 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table multi_capture_4.usertable exists [Sat May 4 11:12:48 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.28462848.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2836.out cli tso query --pd=http://127.0.0.1:2379 ++ date + echo '[Sat May 4 11:12:48 CST 2024] <<<<<< START pulsar cluster in capture_session_done_during_task case >>>>>>' [Sat May 4 11:12:48 CST 2024] <<<<<< START pulsar cluster in capture_session_done_during_task case >>>>>> + workdir=/tmp/tidb_cdc_test/capture_session_done_during_task + cluster_type=normal + cd /tmp/tidb_cdc_test/capture_session_done_during_task + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/capture_session_done_during_task/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/capture_session_done_during_task/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:48 CST 2024] <<<<<< START pulsar cluster in normal mode in capture_session_done_during_task case >>>>>>' [Sat May 4 11:12:48 CST 2024] <<<<<< START pulsar cluster in normal mode in capture_session_done_during_task case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully + set +x + tso='449522530526953475 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522530526953475 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:12:49 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.29392941.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 31-th check, retry later check diff failed 3-th time, retry later wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065c079c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c, pid:1421, start at 2024-05-04 11:12:46.569781943 +0800 CST m=+5.364125001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:46.577 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:46.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:46.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065c079c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c, pid:1421, start at 2024-05-04 11:12:46.569781943 +0800 CST m=+5.364125001 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:46.577 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:46.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:46.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065c09180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c, pid:1494, start at 2024-05-04 11:12:46.701069626 +0800 CST m=+5.443934404 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:46.708 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:46.712 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:46.712 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:50 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2eb14be-518c-4f55-845e-ae8793ef7173 {"id":"b2eb14be-518c-4f55-845e-ae8793ef7173","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792367} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cd3bcc b2eb14be-518c-4f55-845e-ae8793ef7173 /tidb/cdc/default/default/upstream/7364977078021042780 {"id":7364977078021042780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2eb14be-518c-4f55-845e-ae8793ef7173 {"id":"b2eb14be-518c-4f55-845e-ae8793ef7173","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792367} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cd3bcc b2eb14be-518c-4f55-845e-ae8793ef7173 /tidb/cdc/default/default/upstream/7364977078021042780 {"id":7364977078021042780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b2eb14be-518c-4f55-845e-ae8793ef7173 {"id":"b2eb14be-518c-4f55-845e-ae8793ef7173","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792367} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cd3bcc b2eb14be-518c-4f55-845e-ae8793ef7173 /tidb/cdc/default/default/upstream/7364977078021042780 {"id":7364977078021042780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:12:50 CST 2024] <<<<<< START pulsar cluster in processor_resolved_ts_fallback case >>>>>>' [Sat May 4 11:12:50 CST 2024] <<<<<< START pulsar cluster in processor_resolved_ts_fallback case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_resolved_ts_fallback + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:50 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_resolved_ts_fallback case >>>>>>' [Sat May 4 11:12:50 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_resolved_ts_fallback case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar-bookie VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065c11cc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53, pid:1392, start at 2024-05-04 11:12:47.244623931 +0800 CST m=+5.194439474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:47.252 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:47.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:47.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065c11cc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53, pid:1392, start at 2024-05-04 11:12:47.244623931 +0800 CST m=+5.194439474 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:47.252 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:47.219 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:47.219 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065c13500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-ptr6-l8w53, pid:1469, start at 2024-05-04 11:12:47.341752764 +0800 CST m=+5.237830316 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:14:47.349 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:12:47.316 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:02:47.316 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process cdc.test exit for 2-th time... + set +x + tso='449522530795126787 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522530795126787 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:12:50 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.28642866.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 11:12:50 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.73137315.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > check diff successfully ***************** properties ***************** "readallfields"="true" "mysql.port"="4000" "mysql.db"="multi_capture_1" "threadcount"="2" "workload"="core" "operationcount"="0" "dotransactions"="false" "mysql.user"="root" "insertproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "readproportion"="0" "updateproportion"="0" "recordcount"="20" "scanproportion"="0" ********************************************** Run finished, takes 10.229217ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2191.3, Avg(us): 974, Min(us): 489, Max(us): 1551, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "workload"="core" "mysql.db"="multi_capture_2" "recordcount"="20" "operationcount"="0" "readproportion"="0" "requestdistribution"="uniform" "dotransactions"="false" "mysql.port"="4000" "readallfields"="true" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.user"="root" "threadcount"="2" "insertproportion"="0" "updateproportion"="0" ********************************************** Run finished, takes 10.720472ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2136.6, Avg(us): 1011, Min(us): 498, Max(us): 1783, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.port"="4000" "mysql.host"="127.0.0.1" "mysql.user"="root" "scanproportion"="0" "threadcount"="2" "workload"="core" "updateproportion"="0" "requestdistribution"="uniform" "readallfields"="true" "dotransactions"="false" "mysql.db"="multi_capture_3" "readproportion"="0" "insertproportion"="0" "recordcount"="20" "operationcount"="0" ********************************************** Run finished, takes 12.024305ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1829.3, Avg(us): 1026, Min(us): 428, Max(us): 1718, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "updateproportion"="0" "readallfields"="true" "readproportion"="0" "mysql.db"="multi_capture_4" "recordcount"="20" "mysql.host"="127.0.0.1" "operationcount"="0" "insertproportion"="0" "workload"="core" "mysql.user"="root" "scanproportion"="0" "dotransactions"="false" "mysql.port"="4000" "threadcount"="2" "requestdistribution"="uniform" ********************************************** Run finished, takes 12.542133ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1787.0, Avg(us): 1075, Min(us): 504, Max(us): 1767, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:51 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> table ddl_manager.finish_mark not exists for 32-th check, retry later < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:51 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/90912a04-0ff6-443f-88c0-ab38c7d954d0 {"id":"90912a04-0ff6-443f-88c0-ab38c7d954d0","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792368} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c97ecc 90912a04-0ff6-443f-88c0-ab38c7d954d0 /tidb/cdc/default/default/upstream/7364977077313762131 {"id":7364977077313762131,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/90912a04-0ff6-443f-88c0-ab38c7d954d0 {"id":"90912a04-0ff6-443f-88c0-ab38c7d954d0","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792368} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c97ecc 90912a04-0ff6-443f-88c0-ab38c7d954d0 /tidb/cdc/default/default/upstream/7364977077313762131 {"id":7364977077313762131,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/90912a04-0ff6-443f-88c0-ab38c7d954d0 {"id":"90912a04-0ff6-443f-88c0-ab38c7d954d0","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792368} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c97ecc 90912a04-0ff6-443f-88c0-ab38c7d954d0 /tidb/cdc/default/default/upstream/7364977077313762131 + grep -q 'etcd info' {"id":7364977077313762131,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully ++ date + echo '[Sat May 4 11:12:51 CST 2024] <<<<<< START pulsar cluster in owner_resign case >>>>>>' [Sat May 4 11:12:51 CST 2024] <<<<<< START pulsar cluster in owner_resign case >>>>>> + workdir=/tmp/tidb_cdc_test/owner_resign + cluster_type=normal + cd /tmp/tidb_cdc_test/owner_resign + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/owner_resign/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/owner_resign/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/owner_resign/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/owner_resign/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/owner_resign/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/owner_resign/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/owner_resign/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/owner_resign/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/owner_resign/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:51 CST 2024] <<<<<< START pulsar cluster in normal mode in owner_resign case >>>>>>' [Sat May 4 11:12:51 CST 2024] <<<<<< START pulsar cluster in normal mode in owner_resign case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/owner_resign/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/owner_resign/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/owner_resign/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 check diff failed 4-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2818.out cli tso query --pd=http://127.0.0.1:2379 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:52 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9834d572-56d7-419f-81d3-81dd860ee527 {"id":"9834d572-56d7-419f-81d3-81dd860ee527","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792370} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196ca1bf0 9834d572-56d7-419f-81d3-81dd860ee527 /tidb/cdc/default/default/upstream/7364977083349486859 {"id":7364977083349486859,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9834d572-56d7-419f-81d3-81dd860ee527 {"id":"9834d572-56d7-419f-81d3-81dd860ee527","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792370} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196ca1bf0 9834d572-56d7-419f-81d3-81dd860ee527 /tidb/cdc/default/default/upstream/7364977083349486859 {"id":7364977083349486859,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9834d572-56d7-419f-81d3-81dd860ee527 {"id":"9834d572-56d7-419f-81d3-81dd860ee527","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792370} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196ca1bf0 9834d572-56d7-419f-81d3-81dd860ee527 /tidb/cdc/default/default/upstream/7364977083349486859 {"id":7364977083349486859,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:12:52 CST 2024] <<<<<< START pulsar cluster in simple case >>>>>>' [Sat May 4 11:12:52 CST 2024] <<<<<< START pulsar cluster in simple case >>>>>> + workdir=/tmp/tidb_cdc_test/simple + cluster_type=normal + cd /tmp/tidb_cdc_test/simple + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/simple/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/simple/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/simple/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/simple/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/simple/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/simple/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/simple/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/simple/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/simple/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:52 CST 2024] <<<<<< START pulsar cluster in normal mode in simple case >>>>>>' [Sat May 4 11:12:52 CST 2024] <<<<<< START pulsar cluster in normal mode in simple case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/simple/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/simple/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/simple/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff successfully + set +x + tso='449522531596500993 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522531596500993 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:12:53 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.28592861.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:53 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaa1f760-0256-4dc3-9e34-e8f1b99ab7bf {"id":"aaa1f760-0256-4dc3-9e34-e8f1b99ab7bf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792370} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cc25d7 aaa1f760-0256-4dc3-9e34-e8f1b99ab7bf /tidb/cdc/default/default/upstream/7364977086454536417 {"id":7364977086454536417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaa1f760-0256-4dc3-9e34-e8f1b99ab7bf {"id":"aaa1f760-0256-4dc3-9e34-e8f1b99ab7bf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792370} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cc25d7 aaa1f760-0256-4dc3-9e34-e8f1b99ab7bf /tidb/cdc/default/default/upstream/7364977086454536417 {"id":7364977086454536417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/aaa1f760-0256-4dc3-9e34-e8f1b99ab7bf {"id":"aaa1f760-0256-4dc3-9e34-e8f1b99ab7bf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792370} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cc25d7 aaa1f760-0256-4dc3-9e34-e8f1b99ab7bf /tidb/cdc/default/default/upstream/7364977086454536417 {"id":7364977086454536417,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:12:53 CST 2024] <<<<<< START pulsar cluster in resolve_lock case >>>>>>' [Sat May 4 11:12:53 CST 2024] <<<<<< START pulsar cluster in resolve_lock case >>>>>> + workdir=/tmp/tidb_cdc_test/resolve_lock + cluster_type=normal + cd /tmp/tidb_cdc_test/resolve_lock + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/resolve_lock/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/resolve_lock/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/resolve_lock/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/resolve_lock/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/resolve_lock/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/resolve_lock/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/resolve_lock/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/resolve_lock/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/resolve_lock/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:53 CST 2024] <<<<<< START pulsar cluster in normal mode in resolve_lock case >>>>>>' [Sat May 4 11:12:53 CST 2024] <<<<<< START pulsar cluster in normal mode in resolve_lock case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/resolve_lock/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/resolve_lock/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/resolve_lock/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 33-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:54 GMT < Content-Length: 884 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c10f1f5-7d2b-4b45-b144-90e3ab01552b {"id":"0c10f1f5-7d2b-4b45-b144-90e3ab01552b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792371} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3a6b 0c10f1f5-7d2b-4b45-b144-90e3ab01552b /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c10f1f5-7d2b-4b45-b144-90e3ab01552b {"id":"0c10f1f5-7d2b-4b45-b144-90e3ab01552b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792371} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3a6b 0c10f1f5-7d2b-4b45-b144-90e3ab01552b /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0c10f1f5-7d2b-4b45-b144-90e3ab01552b {"id":"0c10f1f5-7d2b-4b45-b144-90e3ab01552b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792371} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3a6b 0c10f1f5-7d2b-4b45-b144-90e3ab01552b /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7362.out cli changefeed create --start-ts=449522520395087873 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-1 Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7364976913536362088,"namespace":"default","id":"changefeed-error-1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:12:54.537899933+08:00","start_ts":449522520395087873,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522520395087873,"checkpoint_ts":449522520395087873,"checkpoint_time":"2024-05-04 11:12:09.388"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process cdc.test exit for 3-th time... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:54 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 public/default \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:12:55 CST 2024] <<<<<< run test case multi_capture success! >>>>>> [Pipeline] // dir [Pipeline] } ++ date + echo '[Sat May 4 11:12:55 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:12:55 CST 2024] <<<<<< pulsar is ready >>>>>> [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Pipeline] // container + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.3345.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [Pipeline] // withEnv + set +x [Pipeline] } [Pipeline] // node [Pipeline] } table ddl_manager.finish_mark not exists for 34-th check, retry later [Pipeline] // podTemplate + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff successfully check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 252 100 252 0 0 1909 0 --:--:-- --:--:-- --:--:-- 1923 + info='{"state":"warning","resolved_ts":449522520788041744,"checkpoint_ts":449522520788041744,"last_warning":{"time":"2024-05-04T11:12:56.06112275+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449522520788041744,"checkpoint_ts":449522520788041744,"last_warning":{"time":"2024-05-04T11:12:56.06112275+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449522520788041744,"checkpoint_ts":449522520788041744,"last_warning":{"time":"2024-05-04T11:12:56.06112275+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":449522520788041744,"checkpoint_ts":449522520788041744,"last_warning":{"time":"2024-05-04T11:12:56.06112275+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' [Pipeline] } [Pipeline] // withEnv [Pipeline] } + /usr/local/pulsar/bin/pulsar-admin namespaces list public + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449522520788041744,"checkpoint_ts":449522520788041744,"last_warning":{"time":"2024-05-04T11:12:56.06112275+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7415.out cli changefeed remove -c changefeed-error-1 [Pipeline] // stage [Pipeline] } + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Sat May 4 11:12:55 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.29532955.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:56 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77223976-0956-4ce1-86ff-b3368d229109 {"id":"77223976-0956-4ce1-86ff-b3368d229109","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792373} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196d4f4d3 77223976-0956-4ce1-86ff-b3368d229109 /tidb/cdc/default/default/upstream/7364977093920204709 {"id":7364977093920204709,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77223976-0956-4ce1-86ff-b3368d229109 {"id":"77223976-0956-4ce1-86ff-b3368d229109","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792373} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196d4f4d3 77223976-0956-4ce1-86ff-b3368d229109 /tidb/cdc/default/default/upstream/7364977093920204709 {"id":7364977093920204709,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/77223976-0956-4ce1-86ff-b3368d229109 {"id":"77223976-0956-4ce1-86ff-b3368d229109","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792373} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196d4f4d3 77223976-0956-4ce1-86ff-b3368d229109 /tidb/cdc/default/default/upstream/7364977093920204709 {"id":7364977093920204709,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:12:56 CST 2024] <<<<<< START pulsar cluster in tiflash case >>>>>>' [Sat May 4 11:12:56 CST 2024] <<<<<< START pulsar cluster in tiflash case >>>>>> + workdir=/tmp/tidb_cdc_test/tiflash + cluster_type=mtls + cd /tmp/tidb_cdc_test/tiflash + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/tiflash/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/tiflash/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/tiflash/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/tiflash/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/tiflash/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus ...............................................+++ ......+++ e is 65537 (0x10001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus ............................................+++ ......+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=/CN=server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus ....................+++ ....................................................................+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=/CN=broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/tiflash/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/tiflash/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem' ++ date + echo '[Sat May 4 11:12:56 CST 2024] <<<<<< START pulsar cluster in mtls mode in tiflash case >>>>>>' [Sat May 4 11:12:56 CST 2024] <<<<<< START pulsar cluster in mtls mode in tiflash case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/tiflash/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/tiflash/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/tiflash/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449522520788041744 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + set +x + tso='449522532634329089 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522532634329089 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:12:57 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.33913393.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 35-th check, retry later Tenant not found Reason: Tenant not found + set +x public/default + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Sat May 4 11:12:58 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:12:58 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 11:12:58 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.33473349.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + i=1 + '[' 1 -gt 20 ']' + sleep 2 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:12:58 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> ++ date + echo '[Sat May 4 11:12:59 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:12:59 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.3297.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-26516?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:12:59 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8db70bef-f14e-4f7a-8984-4c1579375ca4 {"id":"8db70bef-f14e-4f7a-8984-4c1579375ca4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792376} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196d84c2f 8db70bef-f14e-4f7a-8984-4c1579375ca4 /tidb/cdc/default/default/upstream/7364977096315837449 {"id":7364977096315837449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8db70bef-f14e-4f7a-8984-4c1579375ca4 {"id":"8db70bef-f14e-4f7a-8984-4c1579375ca4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792376} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196d84c2f 8db70bef-f14e-4f7a-8984-4c1579375ca4 /tidb/cdc/default/default/upstream/7364977096315837449 {"id":7364977096315837449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8db70bef-f14e-4f7a-8984-4c1579375ca4 {"id":"8db70bef-f14e-4f7a-8984-4c1579375ca4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792376} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196d84c2f 8db70bef-f14e-4f7a-8984-4c1579375ca4 /tidb/cdc/default/default/upstream/7364977096315837449 {"id":7364977096315837449,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:12:59 CST 2024] <<<<<< START pulsar cluster in batch_add_table case >>>>>>' [Sat May 4 11:12:59 CST 2024] <<<<<< START pulsar cluster in batch_add_table case >>>>>> + workdir=/tmp/tidb_cdc_test/batch_add_table + cluster_type=normal + cd /tmp/tidb_cdc_test/batch_add_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/batch_add_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/batch_add_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:12:59 CST 2024] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>>' [Sat May 4 11:12:59 CST 2024] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Sat May 4 11:12:59 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:12:59 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.3409.out cli changefeed create --start-ts=449522530526953475 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-simple-test-8521?protocol=canal-json&enable-tidb-extension=true' public/default wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: 6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 Info: {"upstream_id":7364977078021042780,"namespace":"default","id":"6d748840-5af6-4e29-9b8f-2b5d3ce9cf63","sink_uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-26516?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:12:59.947378032+08:00","start_ts":449522533489180674,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522533489180674,"checkpoint_ts":449522533489180674,"checkpoint_time":"2024-05-04 11:12:59.338"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 11:13:00 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.74797481.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ date + echo '[Sat May 4 11:12:59 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:12:59 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.3339.out cli changefeed create --start-ts=449522530795126787 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-resolve-lock-test-27658?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 17334576-ea8f-4f06-b862-d6f8410c7f1b Info: {"upstream_id":7364977083349486859,"namespace":"default","id":"17334576-ea8f-4f06-b862-d6f8410c7f1b","sink_uri":"pulsar://127.0.0.1:6650/ticdc-simple-test-8521?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:13:00.135064138+08:00","start_ts":449522530526953475,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522530526953475,"checkpoint_ts":449522530526953475,"checkpoint_time":"2024-05-04 11:12:48.038"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 36-th check, retry later + /usr/local/pulsar/bin/pulsar-admin namespaces list public + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:00 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68da0790-32e4-406a-8495-05a198527320 {"id":"68da0790-32e4-406a-8495-05a198527320","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792377} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cc660d 68da0790-32e4-406a-8495-05a198527320 /tidb/cdc/default/default/upstream/7364977088333437948 {"id":7364977088333437948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68da0790-32e4-406a-8495-05a198527320 {"id":"68da0790-32e4-406a-8495-05a198527320","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792377} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cc660d 68da0790-32e4-406a-8495-05a198527320 /tidb/cdc/default/default/upstream/7364977088333437948 {"id":7364977088333437948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68da0790-32e4-406a-8495-05a198527320 {"id":"68da0790-32e4-406a-8495-05a198527320","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792377} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cc660d 68da0790-32e4-406a-8495-05a198527320 /tidb/cdc/default/default/upstream/7364977088333437948 {"id":7364977088333437948,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: ca588a0a-0a48-4824-8535-c25ad0ce40b4 Info: {"upstream_id":7364977086454536417,"namespace":"default","id":"ca588a0a-0a48-4824-8535-c25ad0ce40b4","sink_uri":"pulsar://127.0.0.1:6650/ticdc-resolve-lock-test-27658?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:13:00.901186518+08:00","start_ts":449522530795126787,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522530795126787,"checkpoint_ts":449522530795126787,"checkpoint_time":"2024-05-04 11:12:49.061"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + set +x [Sat May 4 11:13:01 CST 2024] <<<<<< START Pulsar consumer in processor_resolved_ts_fallback case >>>>>> check diff failed 1-th time, retry later + nc -z localhost 6651 + i=3 + '[' 3 -gt 20 ']' + sleep 2 + set +x [Sat May 4 11:13:01 CST 2024] <<<<<< START Pulsar consumer in simple case >>>>>> succeed to verify meta placement rules ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:01 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f52a5a3f-778b-462b-a8e0-02003dce59cc {"id":"f52a5a3f-778b-462b-a8e0-02003dce59cc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792379} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c4140a f52a5a3f-778b-462b-a8e0-02003dce59cc /tidb/cdc/default/default/upstream/7364977070069150205 {"id":7364977070069150205,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f52a5a3f-778b-462b-a8e0-02003dce59cc {"id":"f52a5a3f-778b-462b-a8e0-02003dce59cc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792379} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c4140a f52a5a3f-778b-462b-a8e0-02003dce59cc /tidb/cdc/default/default/upstream/7364977070069150205 {"id":7364977070069150205,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f52a5a3f-778b-462b-a8e0-02003dce59cc {"id":"f52a5a3f-778b-462b-a8e0-02003dce59cc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792379} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c4140a f52a5a3f-778b-462b-a8e0-02003dce59cc /tidb/cdc/default/default/upstream/7364977070069150205 {"id":7364977070069150205,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:13:01 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> + set +x [Sat May 4 11:13:02 CST 2024] <<<<<< START Pulsar consumer in resolve_lock case >>>>>> go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 table ddl_manager.finish_mark not exists for 37-th check, retry later go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/golang/protobuf v1.5.4 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/docker/go-units v0.5.0 go: downloading google.golang.org/protobuf v1.33.0 public/default cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later check diff failed 2-th time, retry later [Sat May 4 11:13:02 CST 2024] <<<<<< START Pulsar consumer in changefeed_finish case >>>>>> [Sat May 4 11:13:02 CST 2024] <<<<<< START Pulsar consumer in capture_session_done_during_task case >>>>>> lease 22318f4196cc660d revoked go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading golang.org/x/sys v0.19.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading golang.org/x/text v0.14.0 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public ++ date + echo '[Sat May 4 11:13:03 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:13:03 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.3435.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-owner-resign-test-10436?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 check diff failed 1-th time, retry later + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table capture_session_done_during_task.t exists + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:03 GMT < Content-Length: 884 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37a92ced-93c0-44dc-bfd5-ebae87137654 {"id":"37a92ced-93c0-44dc-bfd5-ebae87137654","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792380} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3ae6 37a92ced-93c0-44dc-bfd5-ebae87137654 /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37a92ced-93c0-44dc-bfd5-ebae87137654 {"id":"37a92ced-93c0-44dc-bfd5-ebae87137654","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792380} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3ae6 37a92ced-93c0-44dc-bfd5-ebae87137654 /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/37a92ced-93c0-44dc-bfd5-ebae87137654 {"id":"37a92ced-93c0-44dc-bfd5-ebae87137654","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792380} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3ae6 37a92ced-93c0-44dc-bfd5-ebae87137654 /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7535.out cli changefeed create --start-ts=449522520395087873 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-2 check diff failed 1-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7364976913536362088,"namespace":"default","id":"changefeed-error-2","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:13:03.796184696+08:00","start_ts":449522520395087873,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522520395087873,"checkpoint_ts":449522520395087873,"checkpoint_time":"2024-05-04 11:12:09.388"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: bb0b751c-5645-4c4e-9df1-2c42be9bae56 Info: {"upstream_id":7364977077313762131,"namespace":"default","id":"bb0b751c-5645-4c4e-9df1-2c42be9bae56","sink_uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-10436?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:13:04.128119405+08:00","start_ts":449522534562398210,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522534562398210,"checkpoint_ts":449522534562398210,"checkpoint_time":"2024-05-04 11:13:03.432"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 38-th check, retry later table processor_resolved_ts_fallback.t1 exists [Sat May 4 11:13:04 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.34523454.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/otiai10/copy v1.2.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading golang.org/x/time v0.5.0 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading cloud.google.com/go v0.112.2 go: downloading golang.org/x/term v0.19.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/apache/thrift v0.16.0 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 check diff failed 3-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... + set +x [Sat May 4 11:13:05 CST 2024] <<<<<< START Pulsar consumer in owner_resign case >>>>>> + info='{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449522520395087873, "checkpoint_time": "2024-05-04 11:12:09.388", "error": { "time": "2024-05-04T11:13:03.869654146+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522520395087873 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449522520395087873, "checkpoint_time": "2024-05-04 11:12:09.388", "error": { "time": "2024-05-04T11:13:03.869654146+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522520395087873 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449522520395087873, "checkpoint_time": "2024-05-04 11:12:09.388", "error": { "time": "2024-05-04T11:13:03.869654146+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522520395087873 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449522520395087873, '"checkpoint_time":' '"2024-05-04' '11:12:09.388",' '"error":' '{' '"time":' '"2024-05-04T11:13:03.869654146+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522520395087873 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449522520395087873, '"checkpoint_time":' '"2024-05-04' '11:12:09.388",' '"error":' '{' '"time":' '"2024-05-04T11:13:03.869654146+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522520395087873 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522520395087873 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449522520395087873 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7624.out cli changefeed remove -c changefeed-error-2 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:13:05 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' check diff failed 2-th time, retry later Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later check diff failed 2-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449522520395087873 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 39-th check, retry later check diff failed 4-th time, retry later check diff successfully + /usr/local/pulsar/bin/pulsar-admin namespaces list public + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:07 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {UpstreamID:7364977078021042780 Namespace:default ID:6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-26516?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:12:59.947378032 +0800 CST StartTs:449522533489180674 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00152d7a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522533646729217} {CheckpointTs:449522535429308424 MinTableBarrierTs:449522535429308424 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52bdc9d5-42bf-4f96-891a-03d2dc2907ee {"id":"52bdc9d5-42bf-4f96-891a-03d2dc2907ee","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792385} /tidb/cdc/default/__cdc_meta__/capture/b2eb14be-518c-4f55-845e-ae8793ef7173 {"id":"b2eb14be-518c-4f55-845e-ae8793ef7173","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792367} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cd3bcc b2eb14be-518c-4f55-845e-ae8793ef7173 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cd3c8b 52bdc9d5-42bf-4f96-891a-03d2dc2907ee /tidb/cdc/default/default/changefeed/info/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"upstream-id":7364977078021042780,"namespace":"default","changefeed-id":"6d748840-5af6-4e29-9b8f-2b5d3ce9cf63","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-26516?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:12:59.947378032+08:00","start-ts":449522533489180674,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522533646729217} /tidb/cdc/default/default/changefeed/status/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"checkpoint-ts":449522535429308424,"min-table-barrier-ts":449522535429308424,"admin-job-type":0} /tidb/cdc/default/default/task/position/52bdc9d5-42bf-4f96-891a-03d2dc2907ee/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b2eb14be-518c-4f55-845e-ae8793ef7173/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977078021042780 {"id":7364977078021042780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {UpstreamID:7364977078021042780 Namespace:default ID:6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-26516?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:12:59.947378032 +0800 CST StartTs:449522533489180674 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00152d7a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522533646729217} {CheckpointTs:449522535429308424 MinTableBarrierTs:449522535429308424 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52bdc9d5-42bf-4f96-891a-03d2dc2907ee {"id":"52bdc9d5-42bf-4f96-891a-03d2dc2907ee","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792385} /tidb/cdc/default/__cdc_meta__/capture/b2eb14be-518c-4f55-845e-ae8793ef7173 {"id":"b2eb14be-518c-4f55-845e-ae8793ef7173","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792367} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cd3bcc b2eb14be-518c-4f55-845e-ae8793ef7173 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cd3c8b 52bdc9d5-42bf-4f96-891a-03d2dc2907ee /tidb/cdc/default/default/changefeed/info/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"upstream-id":7364977078021042780,"namespace":"default","changefeed-id":"6d748840-5af6-4e29-9b8f-2b5d3ce9cf63","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-26516?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:12:59.947378032+08:00","start-ts":449522533489180674,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522533646729217} /tidb/cdc/default/default/changefeed/status/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"checkpoint-ts":449522535429308424,"min-table-barrier-ts":449522535429308424,"admin-job-type":0} /tidb/cdc/default/default/task/position/52bdc9d5-42bf-4f96-891a-03d2dc2907ee/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b2eb14be-518c-4f55-845e-ae8793ef7173/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977078021042780 {"id":7364977078021042780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {UpstreamID:7364977078021042780 Namespace:default ID:6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-26516?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:12:59.947378032 +0800 CST StartTs:449522533489180674 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00152d7a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522533646729217} {CheckpointTs:449522535429308424 MinTableBarrierTs:449522535429308424 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/52bdc9d5-42bf-4f96-891a-03d2dc2907ee {"id":"52bdc9d5-42bf-4f96-891a-03d2dc2907ee","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792385} /tidb/cdc/default/__cdc_meta__/capture/b2eb14be-518c-4f55-845e-ae8793ef7173 {"id":"b2eb14be-518c-4f55-845e-ae8793ef7173","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792367} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cd3bcc b2eb14be-518c-4f55-845e-ae8793ef7173 /tidb/cdc/default/__cdc_meta__/owner/22318f4196cd3c8b 52bdc9d5-42bf-4f96-891a-03d2dc2907ee /tidb/cdc/default/default/changefeed/info/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"upstream-id":7364977078021042780,"namespace":"default","changefeed-id":"6d748840-5af6-4e29-9b8f-2b5d3ce9cf63","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-26516?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:12:59.947378032+08:00","start-ts":449522533489180674,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522533646729217} /tidb/cdc/default/default/changefeed/status/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"checkpoint-ts":449522535429308424,"min-table-barrier-ts":449522535429308424,"admin-job-type":0} /tidb/cdc/default/default/task/position/52bdc9d5-42bf-4f96-891a-03d2dc2907ee/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/b2eb14be-518c-4f55-845e-ae8793ef7173/6d748840-5af6-4e29-9b8f-2b5d3ce9cf63 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977078021042780 {"id":7364977078021042780,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table owner_resign.t1 exists check diff failed 3-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check diff failed 1-th time, retry later check data successfully public/default ++ date + echo '[Sat May 4 11:13:08 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:13:08 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.3425.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-3194?protocol=canal-json&enable-tidb-extension=true' wait process cdc.test exit for 1-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:13:08 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 40-th check, retry later table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 not exists for 1-th check, retry later go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/felixge/httpsnoop v1.0.4 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:13:08 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... Create changefeed successfully! ID: ff5887c8-37e2-4562-bfac-1c7053e83bc5 Info: {"upstream_id":7364977096315837449,"namespace":"default","id":"ff5887c8-37e2-4562-bfac-1c7053e83bc5","sink_uri":"pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-3194?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:13:09.077962453+08:00","start_ts":449522535869710341,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522535869710341,"checkpoint_ts":449522535869710341,"checkpoint_time":"2024-05-04 11:13:08.419"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:13:09 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.76717673.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:13:09 CST 2024] <<<<<< run test case simple success! >>>>>> check diff failed 5-th time, retry later check diff successfully check diff failed 1-th time, retry later public/default ++ date + echo '[Sat May 4 11:13:10 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:13:10 CST 2024] <<<<<< pulsar is ready >>>>>> table ddl_manager.finish_mark not exists for 41-th check, retry later + set +x [Sat May 4 11:13:10 CST 2024] <<<<<< START Pulsar consumer in batch_add_table case >>>>>> check diff successfully [Sat May 4 11:13:10 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.36783680.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table batch_add_table.finish_mark not exists for 1-th check, retry later Create changefeed successfully! ID: 52f4841d-76b4-4751-857d-0c91161c775f Info: {"upstream_id":7364977093920204709,"namespace":"default","id":"52f4841d-76b4-4751-857d-0c91161c775f","sink_uri":"pulsar+ssl://127.0.0.1:6651/ticdc-tiflash-test-10644?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:13:11.133704678+08:00","start_ts":449522531596500993,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"tls-trust-certs-file-path":"/tmp/tidb_cdc_test/tiflash/ca.cert.pem","connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"auth-tls-certificate-path":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","auth-tls-private-key-path":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522531596500993,"checkpoint_ts":449522531596500993,"checkpoint_time":"2024-05-04 11:12:52.118"} [Sat May 4 11:13:11 CST 2024] <<<<<< START Pulsar consumer in tiflash case >>>>>> table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully check diff successfully find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:13:11 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> killing cdc1: 2844 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 42-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:12 GMT < Content-Length: 884 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/921a0516-8143-4b2e-8273-4148ed8e1e2b {"id":"921a0516-8143-4b2e-8273-4148ed8e1e2b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3b57 921a0516-8143-4b2e-8273-4148ed8e1e2b /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/921a0516-8143-4b2e-8273-4148ed8e1e2b {"id":"921a0516-8143-4b2e-8273-4148ed8e1e2b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3b57 921a0516-8143-4b2e-8273-4148ed8e1e2b /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/921a0516-8143-4b2e-8273-4148ed8e1e2b {"id":"921a0516-8143-4b2e-8273-4148ed8e1e2b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792389} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f41962f3b57 921a0516-8143-4b2e-8273-4148ed8e1e2b /tidb/cdc/default/default/upstream/7364976913536362088 {"id":7364976913536362088,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7727.out cli changefeed create --start-ts=0 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json&enable-tidb-extension=true' -c changefeed-initialize-error Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7364976913536362088,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:13:12.94356513+08:00","start_ts":449522537015017476,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522537015017476,"checkpoint_ts":449522537015017476,"checkpoint_time":"2024-05-04 11:13:12.788"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 2-th time... table batch_add_table.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:13 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {UpstreamID:7364977077313762131 Namespace:default ID:bb0b751c-5645-4c4e-9df1-2c42be9bae56 SinkURI:pulsar://127.0.0.1:6650/ticdc-owner-resign-test-10436?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:13:04.128119405 +0800 CST StartTs:449522534562398210 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037377a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522534732791814} {CheckpointTs:449522536895741960 MinTableBarrierTs:449522536895741960 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/523d8fcf-5624-47e2-a019-d5f7186ceab8 {"id":"523d8fcf-5624-47e2-a019-d5f7186ceab8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792390} /tidb/cdc/default/__cdc_meta__/capture/90912a04-0ff6-443f-88c0-ab38c7d954d0 {"id":"90912a04-0ff6-443f-88c0-ab38c7d954d0","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792368} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c97ecc 90912a04-0ff6-443f-88c0-ab38c7d954d0 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c97fb5 523d8fcf-5624-47e2-a019-d5f7186ceab8 /tidb/cdc/default/default/changefeed/info/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"upstream-id":7364977077313762131,"namespace":"default","changefeed-id":"bb0b751c-5645-4c4e-9df1-2c42be9bae56","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-10436?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:13:04.128119405+08:00","start-ts":449522534562398210,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522534732791814} /tidb/cdc/default/default/changefeed/status/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"checkpoint-ts":449522536895741960,"min-table-barrier-ts":449522536895741960,"admin-job-type":0} /tidb/cdc/default/default/task/position/523d8fcf-5624-47e2-a019-d5f7186ceab8/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/90912a04-0ff6-443f-88c0-ab38c7d954d0/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977077313762131 {"id":7364977077313762131,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {UpstreamID:7364977077313762131 Namespace:default ID:bb0b751c-5645-4c4e-9df1-2c42be9bae56 SinkURI:pulsar://127.0.0.1:6650/ticdc-owner-resign-test-10436?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:13:04.128119405 +0800 CST StartTs:449522534562398210 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037377a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522534732791814} {CheckpointTs:449522536895741960 MinTableBarrierTs:449522536895741960 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/523d8fcf-5624-47e2-a019-d5f7186ceab8 {"id":"523d8fcf-5624-47e2-a019-d5f7186ceab8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792390} /tidb/cdc/default/__cdc_meta__/capture/90912a04-0ff6-443f-88c0-ab38c7d954d0 {"id":"90912a04-0ff6-443f-88c0-ab38c7d954d0","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792368} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c97ecc 90912a04-0ff6-443f-88c0-ab38c7d954d0 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c97fb5 523d8fcf-5624-47e2-a019-d5f7186ceab8 /tidb/cdc/default/default/changefeed/info/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"upstream-id":7364977077313762131,"namespace":"default","changefeed-id":"bb0b751c-5645-4c4e-9df1-2c42be9bae56","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-10436?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:13:04.128119405+08:00","start-ts":449522534562398210,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522534732791814} /tidb/cdc/default/default/changefeed/status/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"checkpoint-ts":449522536895741960,"min-table-barrier-ts":449522536895741960,"admin-job-type":0} /tidb/cdc/default/default/task/position/523d8fcf-5624-47e2-a019-d5f7186ceab8/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/90912a04-0ff6-443f-88c0-ab38c7d954d0/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977077313762131 {"id":7364977077313762131,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {UpstreamID:7364977077313762131 Namespace:default ID:bb0b751c-5645-4c4e-9df1-2c42be9bae56 SinkURI:pulsar://127.0.0.1:6650/ticdc-owner-resign-test-10436?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:13:04.128119405 +0800 CST StartTs:449522534562398210 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037377a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522534732791814} {CheckpointTs:449522536895741960 MinTableBarrierTs:449522536895741960 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/523d8fcf-5624-47e2-a019-d5f7186ceab8 {"id":"523d8fcf-5624-47e2-a019-d5f7186ceab8","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792390} /tidb/cdc/default/__cdc_meta__/capture/90912a04-0ff6-443f-88c0-ab38c7d954d0 {"id":"90912a04-0ff6-443f-88c0-ab38c7d954d0","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792368} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c97ecc 90912a04-0ff6-443f-88c0-ab38c7d954d0 /tidb/cdc/default/__cdc_meta__/owner/22318f4196c97fb5 523d8fcf-5624-47e2-a019-d5f7186ceab8 /tidb/cdc/default/default/changefeed/info/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"upstream-id":7364977077313762131,"namespace":"default","changefeed-id":"bb0b751c-5645-4c4e-9df1-2c42be9bae56","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-10436?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:13:04.128119405+08:00","start-ts":449522534562398210,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522534732791814} /tidb/cdc/default/default/changefeed/status/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"checkpoint-ts":449522536895741960,"min-table-barrier-ts":449522536895741960,"admin-job-type":0} /tidb/cdc/default/default/task/position/523d8fcf-5624-47e2-a019-d5f7186ceab8/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/90912a04-0ff6-443f-88c0-ab38c7d954d0/bb0b751c-5645-4c4e-9df1-2c42be9bae56 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977077313762131 {"id":7364977077313762131,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:13:13 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' table ddl_manager.finish_mark not exists for 43-th check, retry later Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449522537015017476, "checkpoint_time": "2024-05-04 11:13:12.788", "error": { "time": "2024-05-04T11:13:13.117434714+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449522537015017476, "checkpoint_time": "2024-05-04 11:13:12.788", "error": { "time": "2024-05-04T11:13:13.117434714+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449522537015017476, "checkpoint_time": "2024-05-04 11:13:12.788", "error": { "time": "2024-05-04T11:13:13.117434714+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449522537015017476, '"checkpoint_time":' '"2024-05-04' '11:13:12.788",' '"error":' '{' '"time":' '"2024-05-04T11:13:13.117434714+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: pulsar... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:13:15 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> table batch_add_table.finish_mark exists check diff successfully table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:13:16 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 44-th check, retry later Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later check diff successfully wait process cdc.test exit for 1-th time... cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 45-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522538338844674, "checkpoint_time": "2024-05-04 11:13:17.838", "error": null }' + echo '{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522538338844674, "checkpoint_time": "2024-05-04 11:13:17.838", "error": null }' { "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522538338844674, "checkpoint_time": "2024-05-04 11:13:17.838", "error": null } ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449522538338844674, '"checkpoint_time":' '"2024-05-04' '11:13:17.838",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449522538338844674, '"checkpoint_time":' '"2024-05-04' '11:13:17.838",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7860.out cli changefeed pause -c changefeed-initialize-error PASS wait process cdc.test exit for 2-th time... coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 3-th time, retry later wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 11:13:18 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:13:19 CST 2024] <<<<<< run test case tiflash success! >>>>>> + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449522538338844674, "checkpoint_time": "2024-05-04 11:13:17.838", "error": { "time": "2024-05-04T11:13:13.117434714+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449522538338844674, "checkpoint_time": "2024-05-04 11:13:17.838", "error": { "time": "2024-05-04T11:13:13.117434714+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449522538338844674, "checkpoint_time": "2024-05-04 11:13:17.838", "error": { "time": "2024-05-04T11:13:13.117434714+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449522538338844674, '"checkpoint_time":' '"2024-05-04' '11:13:17.838",' '"error":' '{' '"time":' '"2024-05-04T11:13:13.117434714+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state table ddl_manager.finish_mark not exists for 46-th check, retry later + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449522538338844674, '"checkpoint_time":' '"2024-05-04' '11:13:17.838",' '"error":' '{' '"time":' '"2024-05-04T11:13:13.117434714+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7951.out cli changefeed resume -c changefeed-initialize-error check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff failed 4-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } table ddl_manager.finish_mark not exists for 47-th check, retry later [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522538338844674, "checkpoint_time": "2024-05-04 11:13:17.838", "error": null }' + echo '{ "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522538338844674, "checkpoint_time": "2024-05-04 11:13:17.838", "error": null }' { "upstream_id": 7364976913536362088, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449522538338844674, "checkpoint_time": "2024-05-04 11:13:17.838", "error": null } ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449522538338844674, '"checkpoint_time":' '"2024-05-04' '11:13:17.838",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364976913536362088, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449522538338844674, '"checkpoint_time":' '"2024-05-04' '11:13:17.838",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.8039.out cli changefeed remove -c changefeed-initialize-error =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... check diff failed 5-th time, retry later Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449522539649564675 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-13646?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 48-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065e33340003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:12170, start at 2024-05-04 11:13:22.127845188 +0800 CST m=+5.449394512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:22.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:22.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:22.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065e33340003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:12170, start at 2024-05-04 11:13:22.127845188 +0800 CST m=+5.449394512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:22.136 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:22.125 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:22.125 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065e33f4000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:12246, start at 2024-05-04 11:13:22.191792537 +0800 CST m=+5.462268485 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:22.199 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:22.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:22.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later + set +x start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: pulsar... <<================= cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:13:26 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> table ddl_manager.finish_mark not exists for 49-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.13645.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:13:28 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 50-th check, retry later + set +x + tso='449522540826066945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522540826066945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449522540826066945 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sat May 4 11:13:28 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1368213684.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 51-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // timeout check diff successfully [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 2-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:31 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fafc04a1-e4b6-49da-bd40-de59dd9a9e25 {"id":"fafc04a1-e4b6-49da-bd40-de59dd9a9e25","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792409} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41976587d5 fafc04a1-e4b6-49da-bd40-de59dd9a9e25 /tidb/cdc/default/default/upstream/7364977248680358977 {"id":7364977248680358977,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fafc04a1-e4b6-49da-bd40-de59dd9a9e25 {"id":"fafc04a1-e4b6-49da-bd40-de59dd9a9e25","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792409} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41976587d5 fafc04a1-e4b6-49da-bd40-de59dd9a9e25 /tidb/cdc/default/default/upstream/7364977248680358977 {"id":7364977248680358977,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fafc04a1-e4b6-49da-bd40-de59dd9a9e25 {"id":"fafc04a1-e4b6-49da-bd40-de59dd9a9e25","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792409} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41976587d5 fafc04a1-e4b6-49da-bd40-de59dd9a9e25 /tidb/cdc/default/default/upstream/7364977248680358977 {"id":7364977248680358977,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449522540826066945 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.13750.out cli changefeed create --start-ts=449522540826066945 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7364977248680358977,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:13:32.34051606+08:00","start_ts":449522540826066945,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522540826066945,"checkpoint_ts":449522540826066945,"checkpoint_time":"2024-05-04 11:13:27.326"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... table ddl_manager.finish_mark not exists for 52-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:13:32 CST 2024] <<<<<< run test case owner_resign success! >>>>>> Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 1464 0 --:--:-- --:--:-- --:--:-- 1473 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-04 11:13:27.326","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-04 11:13:33.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:13:27.326","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '11:13:33.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:13:27.326","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '11:13:33.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-04 11:13:27.326' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:13:27.326","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '11:13:33.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:13:27.326","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '11:13:33.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-04 11:13:34' + echo 'sink_checkpoint_ts is 2024-05-04' 11:13:27.326 sink_checkpoint_ts is 2024-05-04 11:13:27.326 ++ date -d '2024-05-04 11:13:27.326' +%s + checkpoint_timestamp=1714792407 ++ date -d '2024-05-04 11:13:34' +%s + current_timestamp=1714792414 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' check diff failed 3-th time, retry later + check_table_exists test.t1 127.0.0.1 3306 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065ee4e00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:4700, start at 2024-05-04 11:13:33.535768783 +0800 CST m=+5.149405094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:33.543 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:33.545 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:33.545 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065ee4e00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:4700, start at 2024-05-04 11:13:33.535768783 +0800 CST m=+5.149405094 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:33.543 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:33.545 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:33.545 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065ee7f0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:4777, start at 2024-05-04 11:13:33.704524859 +0800 CST m=+5.264659354 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:33.711 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:33.692 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:33.692 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table test.t1 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 53-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.t1 exists + sleep 5 check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.6145.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 54-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065f1c400013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:8037, start at 2024-05-04 11:13:37.054976993 +0800 CST m=+5.166919705 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:37.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:37.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:37.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065f1c400013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:8037, start at 2024-05-04 11:13:37.054976993 +0800 CST m=+5.166919705 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:37.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:37.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:37.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065f1dc80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:8118, start at 2024-05-04 11:13:37.165357677 +0800 CST m=+5.224048067 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:37.172 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:37.138 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:37.138 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } + set +x + tso='449522543360999425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522543360999425 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Sat May 4 11:13:38 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.61906192.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } check diff failed 5-th time, retry later [Pipeline] // stage table ddl_manager.finish_mark not exists for 55-th check, retry later [Pipeline] } [Sat May 4 11:13:40 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.95709572.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065f2ddc0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr, pid:4884, start at 2024-05-04 11:13:38.195708862 +0800 CST m=+5.623129748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:38.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:38.167 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:38.167 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065f2ddc0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr, pid:4884, start at 2024-05-04 11:13:38.195708862 +0800 CST m=+5.623129748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:38.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:38.167 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:38.167 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065f2e900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-h6bt-23dwr, pid:4953, start at 2024-05-04 11:13:38.216207917 +0800 CST m=+5.588661104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:38.225 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:38.212 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:38.212 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully table ddl_manager.finish_mark not exists for 56-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2264 0 --:--:-- --:--:-- --:--:-- 2271 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-04 11:13:39.726","puller_resolved_ts":"2024-05-04 11:13:34.126","last_synced_ts":"2024-05-04 11:13:34.576","now_ts":"2024-05-04 11:13:41.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-04' '11:13:39.726","puller_resolved_ts":"2024-05-04' '11:13:34.126","last_synced_ts":"2024-05-04' '11:13:34.576","now_ts":"2024-05-04' '11:13:41.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-04' '11:13:39.726","puller_resolved_ts":"2024-05-04' '11:13:34.126","last_synced_ts":"2024-05-04' '11:13:34.576","now_ts":"2024-05-04' '11:13:41.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:41 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9d6f0e4-8f9a-479d-9739-12c17df26363 {"id":"c9d6f0e4-8f9a-479d-9739-12c17df26363","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792418} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41978f67dc c9d6f0e4-8f9a-479d-9739-12c17df26363 /tidb/cdc/default/default/upstream/7364977299867134489 {"id":7364977299867134489,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9d6f0e4-8f9a-479d-9739-12c17df26363 {"id":"c9d6f0e4-8f9a-479d-9739-12c17df26363","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792418} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41978f67dc c9d6f0e4-8f9a-479d-9739-12c17df26363 /tidb/cdc/default/default/upstream/7364977299867134489 {"id":7364977299867134489,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c9d6f0e4-8f9a-479d-9739-12c17df26363 {"id":"c9d6f0e4-8f9a-479d-9739-12c17df26363","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792418} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41978f67dc c9d6f0e4-8f9a-479d-9739-12c17df26363 /tidb/cdc/default/default/upstream/7364977299867134489 {"id":7364977299867134489,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.6337.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:43 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73f2f443-9d3b-4281-8c84-dea5d6fe9a17 {"id":"73f2f443-9d3b-4281-8c84-dea5d6fe9a17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792420} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979b18c1 73f2f443-9d3b-4281-8c84-dea5d6fe9a17 /tidb/cdc/default/default/upstream/7364977314488658432 {"id":7364977314488658432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73f2f443-9d3b-4281-8c84-dea5d6fe9a17 {"id":"73f2f443-9d3b-4281-8c84-dea5d6fe9a17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792420} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979b18c1 73f2f443-9d3b-4281-8c84-dea5d6fe9a17 /tidb/cdc/default/default/upstream/7364977314488658432 {"id":7364977314488658432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73f2f443-9d3b-4281-8c84-dea5d6fe9a17 {"id":"73f2f443-9d3b-4281-8c84-dea5d6fe9a17","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792420} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979b18c1 73f2f443-9d3b-4281-8c84-dea5d6fe9a17 /tidb/cdc/default/default/upstream/7364977314488658432 {"id":7364977314488658432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:13:43 CST 2024] <<<<<< START pulsar cluster in ddl_only_block_related_table case >>>>>>' [Sat May 4 11:13:43 CST 2024] <<<<<< START pulsar cluster in ddl_only_block_related_table case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_only_block_related_table + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_only_block_related_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_only_block_related_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_only_block_related_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_only_block_related_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:13:43 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_only_block_related_table case >>>>>>' [Sat May 4 11:13:43 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_only_block_related_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 57-th check, retry later + set +x + tso='449522545004642306 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522545004642306 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:13:44 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.63786380.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 58-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 59-th check, retry later check diff failed 2-th time, retry later + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:47 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/90863d91-d8c3-49c0-bc82-06cb45c7802a {"id":"90863d91-d8c3-49c0-bc82-06cb45c7802a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979fb0d3 90863d91-d8c3-49c0-bc82-06cb45c7802a /tidb/cdc/default/default/upstream/7364977320016670039 {"id":7364977320016670039,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/90863d91-d8c3-49c0-bc82-06cb45c7802a {"id":"90863d91-d8c3-49c0-bc82-06cb45c7802a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979fb0d3 90863d91-d8c3-49c0-bc82-06cb45c7802a /tidb/cdc/default/default/upstream/7364977320016670039 {"id":7364977320016670039,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/90863d91-d8c3-49c0-bc82-06cb45c7802a {"id":"90863d91-d8c3-49c0-bc82-06cb45c7802a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792425} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979fb0d3 90863d91-d8c3-49c0-bc82-06cb45c7802a /tidb/cdc/default/default/upstream/7364977320016670039 {"id":7364977320016670039,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:13:47 CST 2024] <<<<<< START pulsar cluster in ddl_attributes case >>>>>>' [Sat May 4 11:13:47 CST 2024] <<<<<< START pulsar cluster in ddl_attributes case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_attributes + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_attributes + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_attributes/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_attributes/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_attributes/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_attributes/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_attributes/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_attributes/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_attributes/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_attributes/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_attributes/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_attributes/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_attributes/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_attributes/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:13:47 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_attributes case >>>>>>' [Sat May 4 11:13:47 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_attributes case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_attributes/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_attributes/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_attributes/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 60-th check, retry later check diff failed 3-th time, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 61-th check, retry later check diff failed 4-th time, retry later public/default ++ date + echo '[Sat May 4 11:13:51 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:13:51 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.10041.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json&enable-tidb-extension=true' -c=ddl-only-block-related-table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065fd43c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c, pid:4842, start at 2024-05-04 11:13:48.850762705 +0800 CST m=+5.564427754 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:48.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:48.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:48.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065fd43c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c, pid:4842, start at 2024-05-04 11:13:48.850762705 +0800 CST m=+5.564427754 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:48.860 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:48.865 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:48.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d065fd4d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-dhfz-w0j4c, pid:4916, start at 2024-05-04 11:13:48.879316031 +0800 CST m=+5.542249923 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:48.887 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:48.852 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:48.852 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7364977314488658432,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:13:52.583367394+08:00","start_ts":449522547291324420,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522547291324420,"checkpoint_ts":449522547291324420,"checkpoint_time":"2024-05-04 11:13:51.989"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 11:13:53 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.62606262.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 62-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Sat May 4 11:13:54 CST 2024] <<<<<< START Pulsar consumer in ddl_only_block_related_table case >>>>>> public/default ++ date + echo '[Sat May 4 11:13:54 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:13:54 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.6852.out cli changefeed create --start-ts=449522545004642306 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-attributes-test-25354?protocol=canal-json&enable-tidb-extension=true' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06601f580015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:9188, start at 2024-05-04 11:13:53.669132319 +0800 CST m=+5.348742103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:53.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:53.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:53.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later Create changefeed successfully! ID: 0d541a60-d193-49d5-b228-0dc12b8b0598 Info: {"upstream_id":7364977320016670039,"namespace":"default","id":"0d541a60-d193-49d5-b228-0dc12b8b0598","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-attributes-test-25354?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:13:55.527063702+08:00","start_ts":449522545004642306,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522545004642306,"checkpoint_ts":449522545004642306,"checkpoint_time":"2024-05-04 11:13:43.266"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 63-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06601f580015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:9188, start at 2024-05-04 11:13:53.669132319 +0800 CST m=+5.348742103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:53.676 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:53.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:53.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06602010000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:9270, start at 2024-05-04 11:13:53.684406709 +0800 CST m=+5.302647671 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:53.691 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:53.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:53.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:13:56 GMT < Content-Length: 860 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25210407-2403-4de9-a42d-dc2c34c77b24 {"id":"25210407-2403-4de9-a42d-dc2c34c77b24","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792433} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197cf5035 25210407-2403-4de9-a42d-dc2c34c77b24 /tidb/cdc/default/default/upstream/7364977365793674750 {"id":7364977365793674750,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25210407-2403-4de9-a42d-dc2c34c77b24 {"id":"25210407-2403-4de9-a42d-dc2c34c77b24","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792433} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197cf5035 25210407-2403-4de9-a42d-dc2c34c77b24 /tidb/cdc/default/default/upstream/7364977365793674750 {"id":7364977365793674750,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25210407-2403-4de9-a42d-dc2c34c77b24 {"id":"25210407-2403-4de9-a42d-dc2c34c77b24","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792433} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197cf5035 25210407-2403-4de9-a42d-dc2c34c77b24 /tidb/cdc/default/default/upstream/7364977365793674750 {"id":7364977365793674750,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:13:56 CST 2024] <<<<<< START pulsar cluster in region_merge case >>>>>>' [Sat May 4 11:13:56 CST 2024] <<<<<< START pulsar cluster in region_merge case >>>>>> + workdir=/tmp/tidb_cdc_test/region_merge + cluster_type=mtls + cd /tmp/tidb_cdc_test/region_merge + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/region_merge/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/region_merge/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/region_merge/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/region_merge/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/region_merge/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus .........................+++ .........+++ e is 65537 (0x10001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus ..........................+++ ........................................................................+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=/CN=server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus ..............................................................................................+++ ....................................................................................+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=/CN=broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/region_merge/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/region_merge/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/region_merge/ca.cert.pem' ++ date + echo '[Sat May 4 11:13:57 CST 2024] <<<<<< START pulsar cluster in mtls mode in region_merge case >>>>>>' [Sat May 4 11:13:57 CST 2024] <<<<<< START pulsar cluster in mtls mode in region_merge case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/region_merge/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/region_merge/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/region_merge/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + set +x [Sat May 4 11:13:57 CST 2024] <<<<<< START Pulsar consumer in ddl_attributes case >>>>>> table ddl_manager.finish_mark not exists for 64-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.10592.out cli tso query --pd=http://127.0.0.1:2379 + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 65-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0660743c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2, pid:5015, start at 2024-05-04 11:13:59.077913563 +0800 CST m=+5.303660820 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:59.085 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:59.055 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:59.055 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449522549121875969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522549121875969 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:14:00 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.1063210634.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table ddl_manager.finish_mark not exists for 66-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0660743c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2, pid:5015, start at 2024-05-04 11:13:59.077913563 +0800 CST m=+5.303660820 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:59.085 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:59.055 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:59.055 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066075a00006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2, pid:5097, start at 2024-05-04 11:13:59.150715079 +0800 CST m=+5.323457473 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:15:59.159 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:13:59.144 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:03:59.144 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_only_block_related_table.finish_mark exists wait process 9575 exit for 1-th time... check diff failed 2-th time, retry later public/default + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:03 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05617492-87ef-4022-bf95-27f03dd6024f {"id":"05617492-87ef-4022-bf95-27f03dd6024f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197df30d2 05617492-87ef-4022-bf95-27f03dd6024f /tidb/cdc/default/default/upstream/7364977387210682274 {"id":7364977387210682274,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05617492-87ef-4022-bf95-27f03dd6024f {"id":"05617492-87ef-4022-bf95-27f03dd6024f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197df30d2 05617492-87ef-4022-bf95-27f03dd6024f /tidb/cdc/default/default/upstream/7364977387210682274 {"id":7364977387210682274,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/05617492-87ef-4022-bf95-27f03dd6024f {"id":"05617492-87ef-4022-bf95-27f03dd6024f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792440} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197df30d2 05617492-87ef-4022-bf95-27f03dd6024f /tidb/cdc/default/default/upstream/7364977387210682274 {"id":7364977387210682274,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:14:03 CST 2024] <<<<<< START pulsar cluster in ddl_sequence case >>>>>>' [Sat May 4 11:14:03 CST 2024] <<<<<< START pulsar cluster in ddl_sequence case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_sequence + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_sequence + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_sequence/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_sequence/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_sequence/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_sequence/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_sequence/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_sequence/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_sequence/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_sequence/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_sequence/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_sequence/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_sequence/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_sequence/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:14:03 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_sequence case >>>>>>' [Sat May 4 11:14:03 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_sequence case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_sequence/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_sequence/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_sequence/pulsar-bookie + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process 9575 exit for 2-th time... ++ date + echo '[Sat May 4 11:14:03 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:14:03 CST 2024] <<<<<< pulsar is ready >>>>>> valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Sat May 4 11:14:01 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ true != \n\o ]] + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.62856287.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None table ddl_manager.finish_mark not exists for 67-th check, retry later ++ date + echo '[Sat May 4 11:14:04 CST 2024] <<<<<< START pulsar cluster in processor_etcd_worker_delay case >>>>>>' [Sat May 4 11:14:04 CST 2024] <<<<<< START pulsar cluster in processor_etcd_worker_delay case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_etcd_worker_delay + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_etcd_worker_delay + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_etcd_worker_delay/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_etcd_worker_delay/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_etcd_worker_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_etcd_worker_delay/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:14:04 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_etcd_worker_delay case >>>>>>' [Sat May 4 11:14:04 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_etcd_worker_delay case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process 9575 exit for 3-th time... process 9575 already exit [Sat May 4 11:14:04 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.1017810180.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: df746fd7-9bac-426b-a6f5-fa3370108eff Info: {"upstream_id":7364977365793674750,"namespace":"default","id":"df746fd7-9bac-426b-a6f5-fa3370108eff","sink_uri":"pulsar+ssl://127.0.0.1:6651/ticdc-region-merge-test-590?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:14:04.600588294+08:00","start_ts":449522550404808708,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"tls-trust-certs-file-path":"/tmp/tidb_cdc_test/region_merge/ca.cert.pem","connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"auth-tls-certificate-path":"/tmp/tidb_cdc_test/region_merge/broker_client.cert.pem","auth-tls-private-key-path":"/tmp/tidb_cdc_test/region_merge/broker_client.key-pk8.pem"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522550404808708,"checkpoint_ts":449522550404808708,"checkpoint_time":"2024-05-04 11:14:03.866"} [Sat May 4 11:14:04 CST 2024] <<<<<< START Pulsar consumer in region_merge case >>>>>> check diff failed 3-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 68-th check, retry later split_and_random_merge scale: 20 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:07 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364977314488658432 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:13:52.583367394 +0800 CST StartTs:449522547291324420 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0027bcea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522547435765765} {CheckpointTs:449522551210639367 MinTableBarrierTs:449522551210639367 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a69b95e7-c67d-4133-a397-3fb71de21f7e {"id":"a69b95e7-c67d-4133-a397-3fb71de21f7e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792444} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979b19d1 a69b95e7-c67d-4133-a397-3fb71de21f7e /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364977314488658432,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:13:52.583367394+08:00","start-ts":449522547291324420,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522547435765765} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449522551210639367,"min-table-barrier-ts":449522551210639367,"admin-job-type":0} /tidb/cdc/default/default/task/position/a69b95e7-c67d-4133-a397-3fb71de21f7e/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977314488658432 {"id":7364977314488658432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364977314488658432 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:13:52.583367394 +0800 CST StartTs:449522547291324420 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0027bcea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522547435765765} {CheckpointTs:449522551210639367 MinTableBarrierTs:449522551210639367 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a69b95e7-c67d-4133-a397-3fb71de21f7e {"id":"a69b95e7-c67d-4133-a397-3fb71de21f7e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792444} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979b19d1 a69b95e7-c67d-4133-a397-3fb71de21f7e /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364977314488658432,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:13:52.583367394+08:00","start-ts":449522547291324420,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522547435765765} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449522551210639367,"min-table-barrier-ts":449522551210639367,"admin-job-type":0} /tidb/cdc/default/default/task/position/a69b95e7-c67d-4133-a397-3fb71de21f7e/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977314488658432 {"id":7364977314488658432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364977314488658432 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:13:52.583367394 +0800 CST StartTs:449522547291324420 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0027bcea0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522547435765765} {CheckpointTs:449522551210639367 MinTableBarrierTs:449522551210639367 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522551210639367, checkpointTs: 449522551210639367, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a69b95e7-c67d-4133-a397-3fb71de21f7e {"id":"a69b95e7-c67d-4133-a397-3fb71de21f7e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792444} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979b19d1 a69b95e7-c67d-4133-a397-3fb71de21f7e /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364977314488658432,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:13:52.583367394+08:00","start-ts":449522547291324420,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522547435765765} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449522551210639367,"min-table-barrier-ts":449522551210639367,"admin-job-type":0} /tidb/cdc/default/default/task/position/a69b95e7-c67d-4133-a397-3fb71de21f7e/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977314488658432 {"id":7364977314488658432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 69-th check, retry later check_ts_not_forward ddl-only-block-related-table + nc -z localhost 6650 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public check diff failed 4-th time, retry later public/default table ddl_manager.finish_mark not exists for 70-th check, retry later ++ date + echo '[Sat May 4 11:14:09 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:14:09 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.11097.out cli changefeed create --start-ts=449522549121875969 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-sequence-test-2909?protocol=canal-json&enable-tidb-extension=true' public/default check diff failed 5-th time, retry later ++ date + echo '[Sat May 4 11:14:10 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:14:10 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 11:14:10 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.69166918.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: ef2a6eac-c9a1-4027-af97-3cd90e79229d Info: {"upstream_id":7364977387210682274,"namespace":"default","id":"ef2a6eac-c9a1-4027-af97-3cd90e79229d","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-sequence-test-2909?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:14:10.785758125+08:00","start_ts":449522549121875969,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522549121875969,"checkpoint_ts":449522549121875969,"checkpoint_time":"2024-05-04 11:13:58.972"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 71-th check, retry later + set +x [Sat May 4 11:14:12 CST 2024] <<<<<< START Pulsar consumer in ddl_sequence case >>>>>> check diff successfully wait process cdc.test exit for 1-th time... run task failed 1-th time, retry later wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:13 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed5eb337-03a3-475b-9c6d-9d4d158f437a {"id":"ed5eb337-03a3-475b-9c6d-9d4d158f437a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197f468e9 ed5eb337-03a3-475b-9c6d-9d4d158f437a /tidb/cdc/default/default/upstream/7364977405125618623 {"id":7364977405125618623,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed5eb337-03a3-475b-9c6d-9d4d158f437a {"id":"ed5eb337-03a3-475b-9c6d-9d4d158f437a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197f468e9 ed5eb337-03a3-475b-9c6d-9d4d158f437a /tidb/cdc/default/default/upstream/7364977405125618623 {"id":7364977405125618623,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ed5eb337-03a3-475b-9c6d-9d4d158f437a {"id":"ed5eb337-03a3-475b-9c6d-9d4d158f437a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792450} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4197f468e9 ed5eb337-03a3-475b-9c6d-9d4d158f437a /tidb/cdc/default/default/upstream/7364977405125618623 {"id":7364977405125618623,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 72-th check, retry later wait process cdc.test exit for 3-th time... table ddl_sequence.finish_mark not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:14:14 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> check_ts_not_forward ddl-only-block-related-table [Sat May 4 11:14:14 CST 2024] <<<<<< START Pulsar consumer in processor_etcd_worker_delay case >>>>>> table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 73-th check, retry later *************************** 1. row *************************** count(distinct region_id): 4 table ddl_sequence.finish_mark not exists for 2-th check, retry later table ddl_attributes.finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 74-th check, retry later table ddl_sequence.finish_mark not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 75-th check, retry later table ddl_attributes.finish_mark not exists for 4-th check, retry later split_and_random_merge scale: 40 table ddl_sequence.finish_mark not exists for 4-th check, retry later table processor_delay.t1 exists table processor_delay.t2 not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 5-th check, retry later invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Sat May 4 11:14:21 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> table ddl_manager.finish_mark not exists for 76-th check, retry later table ddl_sequence.finish_mark not exists for 5-th check, retry later table processor_delay.t2 not exists for 2-th check, retry later table ddl_attributes.finish_mark not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 77-th check, retry later table ddl_sequence.finish_mark not exists for 6-th check, retry later table processor_delay.t2 exists table processor_delay.t3 not exists for 1-th check, retry later table ddl_attributes.finish_mark exists check diff successfully table ddl_manager.finish_mark not exists for 78-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table ddl_sequence.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit table ddl_manager.finish_mark not exists for 79-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:14:27 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> [Sat May 4 11:14:27 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> *************************** 1. row *************************** count(distinct region_id): 22 table ddl_manager.finish_mark not exists for 80-th check, retry later table processor_delay.t4 exists table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later table processor_delay.t6 exists table processor_delay.t7 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 81-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... split_and_random_merge scale: 80 table processor_delay.t7 exists table processor_delay.t8 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 82-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table processor_delay.t8 exists table processor_delay.t9 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 83-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table processor_delay.t9 exists table processor_delay.t10 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } table ddl_manager.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) wait process cdc.test exit for 2-th time... [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // node [Pipeline] } Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } table processor_delay.t10 not exists for 2-th check, retry later [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 4-th time... cdc.test: no process found wait process cdc.test exit for 5-th time... process cdc.test already exit [Sat May 4 11:14:40 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t10 exists table processor_delay.t11 exists table processor_delay.t12 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t12 exists table processor_delay.t13 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_changefeed_state http://127.0.0.1:2379 503bb0e0-f8e1-4ffd-b248-684e3d4b2d88 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=503bb0e0-f8e1-4ffd-b248-684e3d4b2d88 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 503bb0e0-f8e1-4ffd-b248-684e3d4b2d88 -s + info='{ "upstream_id": 7364977070069150205, "namespace": "default", "id": "503bb0e0-f8e1-4ffd-b248-684e3d4b2d88", "state": "finished", "checkpoint_tso": 449522557765812230, "checkpoint_time": "2024-05-04 11:14:31.946", "error": null }' + echo '{ "upstream_id": 7364977070069150205, "namespace": "default", "id": "503bb0e0-f8e1-4ffd-b248-684e3d4b2d88", "state": "finished", "checkpoint_tso": 449522557765812230, "checkpoint_time": "2024-05-04 11:14:31.946", "error": null }' { "upstream_id": 7364977070069150205, "namespace": "default", "id": "503bb0e0-f8e1-4ffd-b248-684e3d4b2d88", "state": "finished", "checkpoint_tso": 449522557765812230, "checkpoint_time": "2024-05-04 11:14:31.946", "error": null } ++ echo '{' '"upstream_id":' 7364977070069150205, '"namespace":' '"default",' '"id":' '"503bb0e0-f8e1-4ffd-b248-684e3d4b2d88",' '"state":' '"finished",' '"checkpoint_tso":' 449522557765812230, '"checkpoint_time":' '"2024-05-04' '11:14:31.946",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7364977070069150205, '"namespace":' '"default",' '"id":' '"503bb0e0-f8e1-4ffd-b248-684e3d4b2d88",' '"state":' '"finished",' '"checkpoint_tso":' 449522557765812230, '"checkpoint_time":' '"2024-05-04' '11:14:31.946",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:14:39 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [2024/05/04 11:14:40.762 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/04 11:14:40.762 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7364977086454536417] [2024/05/04 11:14:40.762 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/04 11:14:40.762 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/04 11:14:40.763 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/04 11:14:40.763 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/04 11:14:40.765 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/04 11:14:40.765 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7364977086454536417] [2024/05/04 11:14:40.765 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/04 11:14:40.765 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/04 11:14:40.766 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/04 11:14:40.766 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/04 11:14:40.766 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/04 11:14:40.767 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/04 11:14:40.767 +08:00] [INFO] [store_cache.go:477] ["change store resolve state"] [store=1] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06633ec80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:12256, start at 2024-05-04 11:14:44.827679307 +0800 CST m=+5.315957159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:44.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:44.835 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:44.835 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06633ec80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:12256, start at 2024-05-04 11:14:44.827679307 +0800 CST m=+5.315957159 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:44.838 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:44.835 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:44.835 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06633f780002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:12341, start at 2024-05-04 11:14:44.832573894 +0800 CST m=+5.263159454 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:44.841 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:44.830 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:44.830 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t14 exists table processor_delay.t15 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13693.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06636ecc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:7324, start at 2024-05-04 11:14:47.900242242 +0800 CST m=+5.200295995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:47.909 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:47.909 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:47.909 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06636ecc0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:7324, start at 2024-05-04 11:14:47.900242242 +0800 CST m=+5.200295995 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:47.909 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:47.909 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:47.909 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06636f700007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:7406, start at 2024-05-04 11:14:47.908130727 +0800 CST m=+5.160882575 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:47.915 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:47.900 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:47.900 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t16 not exists for 2-th check, retry later [2024/05/04 11:14:50.769 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/04 11:14:50.769 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/04 11:14:50.769 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/05/04 11:14:50.769 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/05/04 11:14:50.769 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/04 11:14:50.769 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/04 11:14:50.769 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/04 11:14:50.769 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/05/04 11:14:50.769 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/04 11:14:50.769 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/05/04 11:14:50.769 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/04 11:14:50.769 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/05/04 11:14:50.769 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] + set +x + tso='449522562573271041 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522562573271041 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:14:52 CST 2024] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1374213744.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663b13c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:12332, start at 2024-05-04 11:14:52.159368783 +0800 CST m=+5.252760992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:52.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:52.161 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:52.161 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 11:14:52 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.88538855.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t16 exists table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: pulsar... <<================= VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663b13c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:12332, start at 2024-05-04 11:14:52.159368783 +0800 CST m=+5.252760992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:52.166 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:52.161 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:52.161 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0663b2a40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-nrk1-j6b1p, pid:12419, start at 2024-05-04 11:14:52.235486971 +0800 CST m=+5.270692553 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:16:52.241 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:14:52.201 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:04:52.201 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t18 exists table processor_delay.t19 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:56 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/102cde97-1558-42da-9442-1284bccfad56 {"id":"102cde97-1558-42da-9442-1284bccfad56","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792493} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198afbbcb 102cde97-1558-42da-9442-1284bccfad56 /tidb/cdc/default/default/upstream/7364977619103006240 {"id":7364977619103006240,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/102cde97-1558-42da-9442-1284bccfad56 {"id":"102cde97-1558-42da-9442-1284bccfad56","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792493} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198afbbcb 102cde97-1558-42da-9442-1284bccfad56 /tidb/cdc/default/default/upstream/7364977619103006240 {"id":7364977619103006240,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/102cde97-1558-42da-9442-1284bccfad56 {"id":"102cde97-1558-42da-9442-1284bccfad56","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792493} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198afbbcb 102cde97-1558-42da-9442-1284bccfad56 /tidb/cdc/default/default/upstream/7364977619103006240 {"id":7364977619103006240,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:14:56 CST 2024] <<<<<< START pulsar cluster in event_filter case >>>>>>' [Sat May 4 11:14:56 CST 2024] <<<<<< START pulsar cluster in event_filter case >>>>>> + workdir=/tmp/tidb_cdc_test/event_filter + cluster_type=normal + cd /tmp/tidb_cdc_test/event_filter + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/event_filter/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/event_filter/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/event_filter/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/event_filter/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/event_filter/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/event_filter/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/event_filter/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/event_filter/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/event_filter/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/event_filter/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/event_filter/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/event_filter/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:14:56 CST 2024] <<<<<< START pulsar cluster in normal mode in event_filter case >>>>>>' [Sat May 4 11:14:56 CST 2024] <<<<<< START pulsar cluster in normal mode in event_filter case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/event_filter/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/event_filter/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/event_filter/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:14:56 GMT < Content-Length: 860 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4d7f30fd-7125-4318-9ddc-927f799d1ea7 {"id":"4d7f30fd-7125-4318-9ddc-927f799d1ea7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198a382a3 4d7f30fd-7125-4318-9ddc-927f799d1ea7 /tidb/cdc/default/default/upstream/7364977595667734048 {"id":7364977595667734048,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4d7f30fd-7125-4318-9ddc-927f799d1ea7 {"id":"4d7f30fd-7125-4318-9ddc-927f799d1ea7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198a382a3 4d7f30fd-7125-4318-9ddc-927f799d1ea7 /tidb/cdc/default/default/upstream/7364977595667734048 {"id":7364977595667734048,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4d7f30fd-7125-4318-9ddc-927f799d1ea7 {"id":"4d7f30fd-7125-4318-9ddc-927f799d1ea7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792492} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198a382a3 4d7f30fd-7125-4318-9ddc-927f799d1ea7 /tidb/cdc/default/default/upstream/7364977595667734048 {"id":7364977595667734048,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:14:56 CST 2024] <<<<<< START pulsar cluster in cli_with_auth case >>>>>>' [Sat May 4 11:14:56 CST 2024] <<<<<< START pulsar cluster in cli_with_auth case >>>>>> + workdir=/tmp/tidb_cdc_test/cli_with_auth + cluster_type=normal + cd /tmp/tidb_cdc_test/cli_with_auth + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/cli_with_auth/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/cli_with_auth/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/cli_with_auth/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/cli_with_auth/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/cli_with_auth/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/cli_with_auth/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/cli_with_auth/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/cli_with_auth/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/cli_with_auth/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/cli_with_auth/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/cli_with_auth/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/cli_with_auth/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:14:56 CST 2024] <<<<<< START pulsar cluster in normal mode in cli_with_auth case >>>>>>' [Sat May 4 11:14:56 CST 2024] <<<<<< START pulsar cluster in normal mode in cli_with_auth case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/cli_with_auth/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/cli_with_auth/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/cli_with_auth/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } table test.t2 not exists for 1-th check, retry later [Pipeline] // dir [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // timeout + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.13822.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table test.t2 not exists for 2-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + set +x + tso='449522564493737985 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522564493737985 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:14:59 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1386813870.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t20 exists table processor_delay.t21 not exists for 1-th check, retry later table test.t2 not exists for 3-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later public/default table test.t2 not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:02 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a15d4670-0bc5-441d-acc8-c2215b908bd7 {"id":"a15d4670-0bc5-441d-acc8-c2215b908bd7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198c219d5 a15d4670-0bc5-441d-acc8-c2215b908bd7 /tidb/cdc/default/default/upstream/7364977638131018589 {"id":7364977638131018589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a15d4670-0bc5-441d-acc8-c2215b908bd7 {"id":"a15d4670-0bc5-441d-acc8-c2215b908bd7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198c219d5 a15d4670-0bc5-441d-acc8-c2215b908bd7 /tidb/cdc/default/default/upstream/7364977638131018589 {"id":7364977638131018589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a15d4670-0bc5-441d-acc8-c2215b908bd7 {"id":"a15d4670-0bc5-441d-acc8-c2215b908bd7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792499} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4198c219d5 a15d4670-0bc5-441d-acc8-c2215b908bd7 /tidb/cdc/default/default/upstream/7364977638131018589 {"id":7364977638131018589,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:15:02 CST 2024] <<<<<< START pulsar cluster in resourcecontrol case >>>>>>' [Sat May 4 11:15:02 CST 2024] <<<<<< START pulsar cluster in resourcecontrol case >>>>>> + workdir=/tmp/tidb_cdc_test/resourcecontrol + cluster_type=normal + cd /tmp/tidb_cdc_test/resourcecontrol + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/resourcecontrol/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/resourcecontrol/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/resourcecontrol/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resourcecontrol/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/resourcecontrol/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/resourcecontrol/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/resourcecontrol/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/resourcecontrol/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resourcecontrol/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/resourcecontrol/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/resourcecontrol/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/resourcecontrol/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:15:02 CST 2024] <<<<<< START pulsar cluster in normal mode in resourcecontrol case >>>>>>' [Sat May 4 11:15:02 CST 2024] <<<<<< START pulsar cluster in normal mode in resourcecontrol case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/resourcecontrol/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/resourcecontrol/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/resourcecontrol/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Sat May 4 11:15:02 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:15:02 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.9313.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-event-filter-24585?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore ++ date + echo '[Sat May 4 11:15:02 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:15:02 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14208.out cli changefeed create --start-ts=449522562573271041 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-cli-test-27824?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [WARN] --tz is deprecated in changefeed settings. table processor_delay.t22 exists table processor_delay.t23 not exists for 1-th check, retry later Create changefeed successfully! ID: 68ee1192-3962-4c35-b099-0b2db2a17235 Info: {"upstream_id":7364977619103006240,"namespace":"default","id":"68ee1192-3962-4c35-b099-0b2db2a17235","sink_uri":"pulsar://127.0.0.1:6650/ticdc-event-filter-24585?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:15:03.419567402+08:00","start_ts":449522565843255300,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522565843255300,"checkpoint_ts":449522565843255300,"checkpoint_time":"2024-05-04 11:15:02.759"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7364977595667734048,"namespace":"default","id":"custom-changefeed-name","sink_uri":"pulsar://127.0.0.1:6650/ticdc-cli-test-27824?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:15:04.04188384+08:00","start_ts":449522562573271041,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522562573271041,"checkpoint_ts":449522562573271041,"checkpoint_time":"2024-05-04 11:14:50.285"} PASS + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.t2 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Sat May 4 11:15:04 CST 2024] <<<<<< START Pulsar consumer in event_filter case >>>>>> table processor_delay.t23 exists + set +x [Sat May 4 11:15:05 CST 2024] <<<<<< START Pulsar consumer in cli_with_auth case >>>>>> table test.simple not exists for 1-th check, retry later table processor_delay.t24 not exists for 1-th check, retry later table test.t2 not exists for 6-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple exists table test.`simple-dash` not exists for 1-th check, retry later table processor_delay.t24 exists table processor_delay.t25 not exists for 1-th check, retry later table event_filter.t1 does not exists table event_filter.t1 exists table event_filter.t_normal not exists for 1-th check, retry later table test.t2 not exists for 7-th check, retry later ++ date + echo '[Sat May 4 11:15:08 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:15:08 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.14334.out cli changefeed create --start-ts=449522564493737985 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-resourcecontrol-test-1163?protocol=canal-json&enable-tidb-extension=true' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0664abf40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9, pid:4865, start at 2024-05-04 11:15:08.173525662 +0800 CST m=+5.363397805 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:08.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:08.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:08.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 1c280ad2-ef81-46ec-8d24-736f805f4957 Info: {"upstream_id":7364977638131018589,"namespace":"default","id":"1c280ad2-ef81-46ec-8d24-736f805f4957","sink_uri":"pulsar://127.0.0.1:6650/ticdc-resourcecontrol-test-1163?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:15:09.385458+08:00","start_ts":449522564493737985,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522564493737985,"checkpoint_ts":449522564493737985,"checkpoint_time":"2024-05-04 11:14:57.611"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522563006070785, "checkpoint_time": "2024-05-04 11:14:51.936", "error": null }' + echo '{ "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522563006070785, "checkpoint_time": "2024-05-04 11:14:51.936", "error": null }' { "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522563006070785, "checkpoint_time": "2024-05-04 11:14:51.936", "error": null } ++ echo '{' '"upstream_id":' 7364977595667734048, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522563006070785, '"checkpoint_time":' '"2024-05-04' '11:14:51.936",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364977595667734048, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522563006070785, '"checkpoint_time":' '"2024-05-04' '11:14:51.936",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 table test.t2 not exists for 8-th check, retry later changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 + set +x [Sat May 4 11:15:10 CST 2024] <<<<<< START Pulsar consumer in resourcecontrol case >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0664abf40012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9, pid:4865, start at 2024-05-04 11:15:08.173525662 +0800 CST m=+5.363397805 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:08.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:08.157 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:08.157 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0664abe80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-cnnh-t8kt9, pid:4942, start at 2024-05-04 11:15:08.195588257 +0800 CST m=+5.325070535 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:08.202 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:08.204 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:08.204 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table resourcecontrol.finish_mark not exists for 1-th check, retry later Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0018e0e88} {Type:update Path:[Config SyncPointRetention] From: To:0xc0018e0ea0} {Type:update Path:[Config Consistent] From: To:0xc0012aa4d0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14615.out cli changefeed --changefeed-id custom-changefeed-name pause table processor_delay.t26 exists table processor_delay.t27 not exists for 1-th check, retry later table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... table test.t2 not exists for 9-th check, retry later table resourcecontrol.finish_mark exists + set +x check diff successfully [Sat May 4 11:15:13 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.64446446.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later wait process cdc.test exit for 1-th time... table test.t2 not exists for 10-th check, retry later wait process cdc.test exit for 2-th time... table event_filter.t_alter exists wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:15:15 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> table event_filter.finish_mark exists check diff failed 1-th time, retry later table processor_delay.t28 exists table processor_delay.t29 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:16 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ce16da6f-fe11-44ec-952b-7ba353d44bff {"id":"ce16da6f-fe11-44ec-952b-7ba353d44bff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792513} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41990032cc ce16da6f-fe11-44ec-952b-7ba353d44bff /tidb/cdc/default/default/upstream/7364977705934133096 {"id":7364977705934133096,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ce16da6f-fe11-44ec-952b-7ba353d44bff {"id":"ce16da6f-fe11-44ec-952b-7ba353d44bff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792513} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41990032cc ce16da6f-fe11-44ec-952b-7ba353d44bff /tidb/cdc/default/default/upstream/7364977705934133096 {"id":7364977705934133096,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ce16da6f-fe11-44ec-952b-7ba353d44bff {"id":"ce16da6f-fe11-44ec-952b-7ba353d44bff","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792513} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41990032cc ce16da6f-fe11-44ec-952b-7ba353d44bff /tidb/cdc/default/default/upstream/7364977705934133096 {"id":7364977705934133096,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:15:16 CST 2024] <<<<<< START pulsar cluster in force_replicate_table case >>>>>>' [Sat May 4 11:15:16 CST 2024] <<<<<< START pulsar cluster in force_replicate_table case >>>>>> + workdir=/tmp/tidb_cdc_test/force_replicate_table + cluster_type=normal + cd /tmp/tidb_cdc_test/force_replicate_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/force_replicate_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/force_replicate_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/force_replicate_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/force_replicate_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/force_replicate_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/force_replicate_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/force_replicate_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/force_replicate_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/force_replicate_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/force_replicate_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/force_replicate_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/force_replicate_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:15:16 CST 2024] <<<<<< START pulsar cluster in normal mode in force_replicate_table case >>>>>>' [Sat May 4 11:15:16 CST 2024] <<<<<< START pulsar cluster in normal mode in force_replicate_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/force_replicate_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/force_replicate_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/force_replicate_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449522568183414786, "checkpoint_time": "2024-05-04 11:15:11.686", "error": null }' + echo '{ "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449522568183414786, "checkpoint_time": "2024-05-04 11:15:11.686", "error": null }' { "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449522568183414786, "checkpoint_time": "2024-05-04 11:15:11.686", "error": null } ++ echo '{' '"upstream_id":' 7364977595667734048, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449522568183414786, '"checkpoint_time":' '"2024-05-04' '11:15:11.686",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7364977595667734048, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449522568183414786, '"checkpoint_time":' '"2024-05-04' '11:15:11.686",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14694.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name table test.t2 not exists for 11-th check, retry later Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc003953fc8} {Type:update Path:[Config SyncPointRetention] From: To:0xc003953fd8} {Type:update Path:[Config Consistent] From: To:0xc0014a1490} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7364977595667734048,"namespace":"default","id":"custom-changefeed-name","sink_uri":"pulsar://127.0.0.1:6650/ticdc-cli-test-27824?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:15:04.04188384+08:00","start_ts":449522562573271041,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":0,"checkpoint_ts":449522568183414786,"checkpoint_time":"2024-05-04 11:15:11.686"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 check diff failed 2-th time, retry later table test.t2 not exists for 12-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14739.out cli changefeed --changefeed-id custom-changefeed-name resume table processor_delay.t30 exists table processor_delay.t31 not exists for 1-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff successfully wait process cdc.test exit for 1-th time... table test.t2 exists check diff successfully + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process cdc.test exit for 2-th time... wait process cdc.test exit for 1-th time... + set +x wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:15:21 CST 2024] <<<<<< run test case event_filter success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:15:21 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later pass check, checkpoint tso not forward after 10s run task successfully wait process 10183 exit for 1-th time... wait process 10183 exit for 2-th time... wait process 10183 exit for 3-th time... wait process 10183 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (10183) - No such process wait process 10183 exit for 5-th time... process 10183 already exit [Sat May 4 11:15:15 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.1089110893.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:19 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364977314488658432 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:13:52.583367394 +0800 CST StartTs:449522547291324420 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002468a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522547435765765} {CheckpointTs:449522551367663629 MinTableBarrierTs:449522569849077764 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9089d6de-30e0-4394-aac2-33fffeaa278d {"id":"9089d6de-30e0-4394-aac2-33fffeaa278d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792516} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979b1bf4 9089d6de-30e0-4394-aac2-33fffeaa278d /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364977314488658432,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:13:52.583367394+08:00","start-ts":449522547291324420,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522547435765765} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449522551367663629,"min-table-barrier-ts":449522569849077764,"admin-job-type":0} /tidb/cdc/default/default/task/position/9089d6de-30e0-4394-aac2-33fffeaa278d/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977314488658432 {"id":7364977314488658432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364977314488658432 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:13:52.583367394 +0800 CST StartTs:449522547291324420 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002468a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522547435765765} {CheckpointTs:449522551367663629 MinTableBarrierTs:449522569849077764 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9089d6de-30e0-4394-aac2-33fffeaa278d {"id":"9089d6de-30e0-4394-aac2-33fffeaa278d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792516} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979b1bf4 9089d6de-30e0-4394-aac2-33fffeaa278d /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364977314488658432,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:13:52.583367394+08:00","start-ts":449522547291324420,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522547435765765} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449522551367663629,"min-table-barrier-ts":449522569849077764,"admin-job-type":0} /tidb/cdc/default/default/task/position/9089d6de-30e0-4394-aac2-33fffeaa278d/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977314488658432 {"id":7364977314488658432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7364977314488658432 Namespace:default ID:ddl-only-block-related-table SinkURI:pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:13:52.583367394 +0800 CST StartTs:449522547291324420 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc002468a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522547435765765} {CheckpointTs:449522551367663629 MinTableBarrierTs:449522569849077764 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449522551367663629, checkpointTs: 449522551367663629, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9089d6de-30e0-4394-aac2-33fffeaa278d {"id":"9089d6de-30e0-4394-aac2-33fffeaa278d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792516} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f41979b1bf4 9089d6de-30e0-4394-aac2-33fffeaa278d /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7364977314488658432,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"pulsar://127.0.0.1:6650/ticdc-common-1-test-1849?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:13:52.583367394+08:00","start-ts":449522547291324420,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522547435765765} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449522551367663629,"min-table-barrier-ts":449522569849077764,"admin-job-type":0} /tidb/cdc/default/default/task/position/9089d6de-30e0-4394-aac2-33fffeaa278d/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977314488658432 + grep -q 'etcd info' {"id":7364977314488658432,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 1-th time, retry later check diff successfully check_ts_forward ddl-only-block-related-table changefeed is working normally rts: 449522570635509760->449522570897391600 checkpoint: 449522570635509760->449522570897391600 run task successfully table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later public/default wait process cdc.test exit for 1-th time... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522571328880643, "checkpoint_time": "2024-05-04 11:15:23.685", "error": null }' + echo '{ "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522571328880643, "checkpoint_time": "2024-05-04 11:15:23.685", "error": null }' { "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522571328880643, "checkpoint_time": "2024-05-04 11:15:23.685", "error": null } ++ echo '{' '"upstream_id":' 7364977595667734048, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522571328880643, '"checkpoint_time":' '"2024-05-04' '11:15:23.685",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364977595667734048, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522571328880643, '"checkpoint_time":' '"2024-05-04' '11:15:23.685",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14832.out cli changefeed --changefeed-id custom-changefeed-name remove ++ date + echo '[Sat May 4 11:15:24 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:15:24 CST 2024] <<<<<< pulsar is ready >>>>>> wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:15:25 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> Create changefeed successfully! ID: ad9bac6a-da9d-4f6a-86f1-f5bc0856f617 Info: {"upstream_id":7364977705934133096,"namespace":"default","id":"ad9bac6a-da9d-4f6a-86f1-f5bc0856f617","sink_uri":"pulsar://127.0.0.1:6650/ticdc-force_replicate_table-test-25247?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:15:25.204012088+08:00","start_ts":449522568582397954,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522568582397954,"checkpoint_ts":449522568582397954,"checkpoint_time":"2024-05-04 11:15:13.208"} [Sat May 4 11:15:25 CST 2024] <<<<<< START Pulsar consumer in force_replicate_table case >>>>>> \033[0;36m<<< Run all test success >>>\033[0m Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449522571328880643 SinkURI: pulsar://127.0.0.1:6650/ticdc-cli-test-27824?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x table force_replicate_table.t0 not exists for 1-th check, retry later table processor_delay.t34 exists table processor_delay.t35 not exists for 1-th check, retry later table force_replicate_table.t0 exists table force_replicate_table.t1 not exists for 1-th check, retry later table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later table force_replicate_table.t1 exists table force_replicate_table.t2 not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14905.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-cli-test-27824?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7364977595667734048,"namespace":"default","id":"custom-changefeed-name","sink_uri":"pulsar://127.0.0.1:6650/ticdc-cli-test-27824?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:15:31.659046948+08:00","start_ts":449522573386973187,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522573386973187,"checkpoint_ts":449522573386973187,"checkpoint_time":"2024-05-04 11:15:31.536"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table processor_delay.t36 exists table processor_delay.t37 not exists for 1-th check, retry later table force_replicate_table.t2 exists table force_replicate_table.t3 not exists for 1-th check, retry later + set +x table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table force_replicate_table.t3 exists table force_replicate_table.t4 not exists for 1-th check, retry later table processor_delay.t38 exists table processor_delay.t39 not exists for 1-th check, retry later Verifying downstream PD is started... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522574474608644, "checkpoint_time": "2024-05-04 11:15:35.685", "error": null }' + echo '{ "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522574474608644, "checkpoint_time": "2024-05-04 11:15:35.685", "error": null }' { "upstream_id": 7364977595667734048, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449522574474608644, "checkpoint_time": "2024-05-04 11:15:35.685", "error": null } ++ echo '{' '"upstream_id":' 7364977595667734048, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522574474608644, '"checkpoint_time":' '"2024-05-04' '11:15:35.685",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364977595667734048, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449522574474608644, '"checkpoint_time":' '"2024-05-04' '11:15:35.685",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.15021.out cli unsafe delete-service-gc-safepoint Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.15054.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table processor_delay.t40 exists table processor_delay.t41 not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 table processor_delay.t42 exists table processor_delay.t43 not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.15150.out cli unsafe resolve-lock --region=78 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.15186.out cli unsafe resolve-lock --region=78 --ts=449522575863971843 table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=19,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666f6380004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt, pid:21600, start at 2024-05-04 11:15:45.679321672 +0800 CST m=+5.130984116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.686 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666f6380004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt, pid:21600, start at 2024-05-04 11:15:45.679321672 +0800 CST m=+5.130984116 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.686 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.678 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.678 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666f9400009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt, pid:21664, start at 2024-05-04 11:15:45.883774823 +0800 CST m=+5.287408519 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.890 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.872 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.872 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fa940003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10695, start at 2024-05-04 11:15:45.958552213 +0800 CST m=+5.260093566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 137k 3989 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" table processor_delay.t44 exists table processor_delay.t45 not exists for 1-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.23089.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fa940003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10695, start at 2024-05-04 11:15:45.958552213 +0800 CST m=+5.260093566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fb440013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10786, start at 2024-05-04 11:15:46.031079689 +0800 CST m=+5.273318153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:46.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:46.001 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:46.001 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 run task successfully table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later + set +x + tso='449522578011717633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522578011717633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x wait process cdc.test exit for 1-th time... ***************** properties ***************** "insertproportion"="0" "mysql.db"="move_table" "scanproportion"="0" "recordcount"="10000" "dotransactions"="false" "workload"="core" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "mysql.port"="4000" "operationcount"="0" "readallfields"="true" "mysql.user"="root" "readproportion"="0" "threadcount"="10" "updateproportion"="0" ********************************************** } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 197k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v8.2.0-alpha-54-gb297eae37", "git_hash": "b297eae37cdb06fa18bc5388b04d2564a81a0b87", "id": "fa66ce11-bfb2-48ec-b953-8d436fe19749", "pid": 13747, "is_owner": true + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.12177.out cli tso query --pd=http://127.0.0.1:2379 }wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table processor_delay.t46 exists table processor_delay.t47 not exists for 1-th check, retry later Run finished, takes 1.301874527s INSERT - Takes(s): 1.3, Count: 10000, OPS: 7706.1, Avg(us): 1254, Min(us): 871, Max(us): 6064, 95th(us): 2000, 99th(us): 2000 [Sat May 4 11:15:52 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2318123183.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:15:52 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:15:52 CST 2024] <<<<<< run test case cli_with_auth success! >>>>>> + set +x + tso='449522578596036609 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522578596036609 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449522578596036609 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Sat May 4 11:15:52 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1221712219.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066758e40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:12264, start at 2024-05-04 11:15:52.01474584 +0800 CST m=+5.228805541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:52.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:51.993 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:51.993 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066758e40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:12264, start at 2024-05-04 11:15:52.01474584 +0800 CST m=+5.228805541 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:52.021 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:51.993 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:51.993 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06675a5c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-f97z-fbrb7, pid:12348, start at 2024-05-04 11:15:52.090003838 +0800 CST m=+5.250239395 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:52.096 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:52.087 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:52.087 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:55 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53862299-39a5-46fb-ae49-2b4a19465615 {"id":"53862299-39a5-46fb-ae49-2b4a19465615","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792552} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938df2 53862299-39a5-46fb-ae49-2b4a19465615 /tidb/cdc/default/default/upstream/7364977857335688458 {"id":7364977857335688458,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53862299-39a5-46fb-ae49-2b4a19465615 {"id":"53862299-39a5-46fb-ae49-2b4a19465615","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792552} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938df2 53862299-39a5-46fb-ae49-2b4a19465615 /tidb/cdc/default/default/upstream/7364977857335688458 {"id":7364977857335688458,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53862299-39a5-46fb-ae49-2b4a19465615 {"id":"53862299-39a5-46fb-ae49-2b4a19465615","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792552} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938df2 53862299-39a5-46fb-ae49-2b4a19465615 /tidb/cdc/default/default/upstream/7364977857335688458 {"id":7364977857335688458,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:15:55 CST 2024] <<<<<< START pulsar cluster in move_table case >>>>>>' [Sat May 4 11:15:55 CST 2024] <<<<<< START pulsar cluster in move_table case >>>>>> + workdir=/tmp/tidb_cdc_test/move_table + cluster_type=normal + cd /tmp/tidb_cdc_test/move_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/move_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/move_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/move_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/move_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/move_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/move_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/move_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/move_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/move_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/move_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/move_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/move_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:15:55 CST 2024] <<<<<< START pulsar cluster in normal mode in move_table case >>>>>>' [Sat May 4 11:15:55 CST 2024] <<<<<< START pulsar cluster in normal mode in move_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/move_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/move_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/move_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table processor_delay.t48 exists table processor_delay.t49 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:15:55 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac7c66b5-acd1-415b-92b1-64a89a224b8a {"id":"ac7c66b5-acd1-415b-92b1-64a89a224b8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199924ad6 ac7c66b5-acd1-415b-92b1-64a89a224b8a /tidb/cdc/default/default/upstream/7364977868863157362 {"id":7364977868863157362,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac7c66b5-acd1-415b-92b1-64a89a224b8a {"id":"ac7c66b5-acd1-415b-92b1-64a89a224b8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199924ad6 ac7c66b5-acd1-415b-92b1-64a89a224b8a /tidb/cdc/default/default/upstream/7364977868863157362 {"id":7364977868863157362,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ac7c66b5-acd1-415b-92b1-64a89a224b8a {"id":"ac7c66b5-acd1-415b-92b1-64a89a224b8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199924ad6 ac7c66b5-acd1-415b-92b1-64a89a224b8a /tidb/cdc/default/default/upstream/7364977868863157362 {"id":7364977868863157362,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449522578596036609 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.12277.out cli changefeed create --start-ts=449522578596036609 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7364977868863157362,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:15:56.411058083+08:00","start_ts":449522578596036609,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522578596036609,"checkpoint_ts":449522578596036609,"checkpoint_time":"2024-05-04 11:15:51.407"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ***************** properties ***************** "recordcount"="20" "mysql.db"="changefeed_auto_stop_1" "insertproportion"="0" "mysql.port"="4000" "scanproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "threadcount"="4" "requestdistribution"="uniform" "readallfields"="true" "mysql.user"="root" "updateproportion"="0" "operationcount"="0" "workload"="core" "readproportion"="0" ********************************************** Run finished, takes 9.020421ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3767.7, Avg(us): 1696, Min(us): 1084, Max(us): 3735, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "dotransactions"="false" "workload"="core" "mysql.user"="root" "readallfields"="true" "updateproportion"="0" "mysql.db"="changefeed_auto_stop_2" "mysql.host"="127.0.0.1" "threadcount"="4" "mysql.port"="4000" "recordcount"="20" "readproportion"="0" "insertproportion"="0" "requestdistribution"="uniform" "scanproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 11.350459ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2843.3, Avg(us): 2129, Min(us): 1259, Max(us): 4278, 95th(us): 5000, 99th(us): 5000 table processor_delay.t49 exists + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 ***************** properties ***************** "mysql.db"="changefeed_auto_stop_3" "scanproportion"="0" "updateproportion"="0" "readallfields"="true" "mysql.user"="root" "recordcount"="20" "readproportion"="0" "requestdistribution"="uniform" "threadcount"="4" "dotransactions"="false" "operationcount"="0" "mysql.port"="4000" "workload"="core" "mysql.host"="127.0.0.1" "insertproportion"="0" ********************************************** Run finished, takes 9.98605ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3739.2, Avg(us): 1890, Min(us): 1065, Max(us): 4534, 95th(us): 5000, 99th(us): 5000 table processor_delay.t50 not exists for 1-th check, retry later ***************** properties ***************** "requestdistribution"="uniform" "readproportion"="0" "insertproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="20" "mysql.port"="4000" "updateproportion"="0" "threadcount"="4" "workload"="core" "mysql.user"="root" "mysql.db"="changefeed_auto_stop_4" "dotransactions"="false" "readallfields"="true" "scanproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 10.526377ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3204.1, Avg(us): 1897, Min(us): 997, Max(us): 4678, 95th(us): 5000, 99th(us): 5000 [Sat May 4 11:15:58 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS= + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1388613888.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + grep -q 'failed to get info:' + echo '' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + '[' pulsar == mysql ']' + stop_tidb_cluster + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table processor_delay.t50 exists check diff failed 1-th time, retry later public/default + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:01 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93611742-043f-49e1-be14-aa2477ef4063 {"id":"93611742-043f-49e1-be14-aa2477ef4063","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792558} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199adf834 93611742-043f-49e1-be14-aa2477ef4063 /tidb/cdc/default/default/upstream/7364977893224571227 {"id":7364977893224571227,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93611742-043f-49e1-be14-aa2477ef4063 {"id":"93611742-043f-49e1-be14-aa2477ef4063","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792558} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199adf834 93611742-043f-49e1-be14-aa2477ef4063 /tidb/cdc/default/default/upstream/7364977893224571227 {"id":7364977893224571227,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/93611742-043f-49e1-be14-aa2477ef4063 {"id":"93611742-043f-49e1-be14-aa2477ef4063","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792558} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199adf834 93611742-043f-49e1-be14-aa2477ef4063 /tidb/cdc/default/default/upstream/7364977893224571227 {"id":7364977893224571227,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:16:01 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1394313945.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ date + echo '[Sat May 4 11:16:01 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:16:01 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.23647.out cli changefeed create --start-ts=449522578011717633 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 41053777-e1a1-47f8-a8ef-ac0f4645f6b1 Info: {"upstream_id":7364977857335688458,"namespace":"default","id":"41053777-e1a1-47f8-a8ef-ac0f4645f6b1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:16:02.455842566+08:00","start_ts":449522578011717633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522578011717633,"checkpoint_ts":449522578011717633,"checkpoint_time":"2024-05-04 11:15:49.178"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2636 0 --:--:-- --:--:-- --:--:-- 2662 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-04 11:15:49.875","puller_resolved_ts":"2024-05-04 11:15:42.825","last_synced_ts":"2024-05-04 11:13:34.576","now_ts":"2024-05-04 11:15:51.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '11:15:49.875","puller_resolved_ts":"2024-05-04' '11:15:42.825","last_synced_ts":"2024-05-04' '11:13:34.576","now_ts":"2024-05-04' '11:15:51.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 11347 8.0 0.0 13783828 142696 ? Sl 11:13 0:12 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 11406 5.4 0.0 13520596 138464 ? Sl 11:13 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714792557.171072,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0034f2380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714792557.171136,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714792557.2344992,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00237a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714792557.2345905,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714792557.8759627,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022628c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714792557.8760343,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-04T11:16:01.740672+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e4a700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:01.741311+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00143cfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:01.798415+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010d41c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} wait process cdc.test exit for 2-th time... + set +x [Sat May 4 11:16:03 CST 2024] <<<<<< START Pulsar consumer in move_table case >>>>>> [Sat May 4 11:16:03 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2370823711.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:04 GMT < Content-Length: 1273 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c39f47c-2b1e-43f6-8ef0-b78cc6248397 {"id":"5c39f47c-2b1e-43f6-8ef0-b78cc6248397","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792561} /tidb/cdc/default/__cdc_meta__/capture/93611742-043f-49e1-be14-aa2477ef4063 {"id":"93611742-043f-49e1-be14-aa2477ef4063","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792558} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199adf834 93611742-043f-49e1-be14-aa2477ef4063 /tidb/cdc/default/__cdc_meta__/owner/22318f4199adf867 5c39f47c-2b1e-43f6-8ef0-b78cc6248397 /tidb/cdc/default/default/upstream/7364977893224571227 {"id":7364977893224571227,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c39f47c-2b1e-43f6-8ef0-b78cc6248397 {"id":"5c39f47c-2b1e-43f6-8ef0-b78cc6248397","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792561} /tidb/cdc/default/__cdc_meta__/capture/93611742-043f-49e1-be14-aa2477ef4063 {"id":"93611742-043f-49e1-be14-aa2477ef4063","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792558} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199adf834 93611742-043f-49e1-be14-aa2477ef4063 /tidb/cdc/default/__cdc_meta__/owner/22318f4199adf867 5c39f47c-2b1e-43f6-8ef0-b78cc6248397 /tidb/cdc/default/default/upstream/7364977893224571227 {"id":7364977893224571227,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5c39f47c-2b1e-43f6-8ef0-b78cc6248397 {"id":"5c39f47c-2b1e-43f6-8ef0-b78cc6248397","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792561} /tidb/cdc/default/__cdc_meta__/capture/93611742-043f-49e1-be14-aa2477ef4063 {"id":"93611742-043f-49e1-be14-aa2477ef4063","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792558} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199adf834 93611742-043f-49e1-be14-aa2477ef4063 /tidb/cdc/default/__cdc_meta__/owner/22318f4199adf867 5c39f47c-2b1e-43f6-8ef0-b78cc6248397 /tidb/cdc/default/default/upstream/7364977893224571227 {"id":7364977893224571227,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:16:04 CST 2024] <<<<<< START pulsar cluster in changefeed_auto_stop case >>>>>>' [Sat May 4 11:16:04 CST 2024] <<<<<< START pulsar cluster in changefeed_auto_stop case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_auto_stop + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_auto_stop + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_auto_stop/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_auto_stop/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_auto_stop/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_auto_stop/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_auto_stop/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_auto_stop/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_auto_stop/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_auto_stop/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:16:04 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_auto_stop case >>>>>>' [Sat May 4 11:16:04 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_auto_stop case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_auto_stop/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_auto_stop/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_auto_stop/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:04 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {UpstreamID:7364977857335688458 Namespace:default ID:41053777-e1a1-47f8-a8ef-ac0f4645f6b1 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:16:02.455842566 +0800 CST StartTs:449522578011717633 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00221c5a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522581485387780} {CheckpointTs:449522578444517529 MinTableBarrierTs:449522582592684033 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2528bb96-79de-4bcd-a656-9d2b6a590ddb {"id":"2528bb96-79de-4bcd-a656-9d2b6a590ddb","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792564} /tidb/cdc/default/__cdc_meta__/capture/53862299-39a5-46fb-ae49-2b4a19465615 {"id":"53862299-39a5-46fb-ae49-2b4a19465615","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792552} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938df2 53862299-39a5-46fb-ae49-2b4a19465615 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938e6e 2528bb96-79de-4bcd-a656-9d2b6a590ddb /tidb/cdc/default/default/changefeed/info/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"upstream-id":7364977857335688458,"namespace":"default","changefeed-id":"41053777-e1a1-47f8-a8ef-ac0f4645f6b1","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:16:02.455842566+08:00","start-ts":449522578011717633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522581485387780} /tidb/cdc/default/default/changefeed/status/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":449522578444517529,"min-table-barrier-ts":449522582592684033,"admin-job-type":0} /tidb/cdc/default/default/task/position/2528bb96-79de-4bcd-a656-9d2b6a590ddb/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/53862299-39a5-46fb-ae49-2b4a19465615/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977857335688458 {"id":7364977857335688458,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {UpstreamID:7364977857335688458 Namespace:default ID:41053777-e1a1-47f8-a8ef-ac0f4645f6b1 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:16:02.455842566 +0800 CST StartTs:449522578011717633 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00221c5a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522581485387780} {CheckpointTs:449522578444517529 MinTableBarrierTs:449522582592684033 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2528bb96-79de-4bcd-a656-9d2b6a590ddb {"id":"2528bb96-79de-4bcd-a656-9d2b6a590ddb","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792564} /tidb/cdc/default/__cdc_meta__/capture/53862299-39a5-46fb-ae49-2b4a19465615 {"id":"53862299-39a5-46fb-ae49-2b4a19465615","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792552} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938df2 53862299-39a5-46fb-ae49-2b4a19465615 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938e6e 2528bb96-79de-4bcd-a656-9d2b6a590ddb /tidb/cdc/default/default/changefeed/info/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"upstream-id":7364977857335688458,"namespace":"default","changefeed-id":"41053777-e1a1-47f8-a8ef-ac0f4645f6b1","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:16:02.455842566+08:00","start-ts":449522578011717633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522581485387780} /tidb/cdc/default/default/changefeed/status/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":449522578444517529,"min-table-barrier-ts":449522582592684033,"admin-job-type":0} /tidb/cdc/default/default/task/position/2528bb96-79de-4bcd-a656-9d2b6a590ddb/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/53862299-39a5-46fb-ae49-2b4a19465615/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977857335688458 {"id":7364977857335688458,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {UpstreamID:7364977857335688458 Namespace:default ID:41053777-e1a1-47f8-a8ef-ac0f4645f6b1 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:16:02.455842566 +0800 CST StartTs:449522578011717633 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00221c5a0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522581485387780} {CheckpointTs:449522578444517529 MinTableBarrierTs:449522582592684033 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2528bb96-79de-4bcd-a656-9d2b6a590ddb {"id":"2528bb96-79de-4bcd-a656-9d2b6a590ddb","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792564} /tidb/cdc/default/__cdc_meta__/capture/53862299-39a5-46fb-ae49-2b4a19465615 {"id":"53862299-39a5-46fb-ae49-2b4a19465615","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792552} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938df2 53862299-39a5-46fb-ae49-2b4a19465615 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938e6e 2528bb96-79de-4bcd-a656-9d2b6a590ddb /tidb/cdc/default/default/changefeed/info/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"upstream-id":7364977857335688458,"namespace":"default","changefeed-id":"41053777-e1a1-47f8-a8ef-ac0f4645f6b1","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:16:02.455842566+08:00","start-ts":449522578011717633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522581485387780} /tidb/cdc/default/default/changefeed/status/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":449522578444517529,"min-table-barrier-ts":449522582592684033,"admin-job-type":0} /tidb/cdc/default/default/task/position/2528bb96-79de-4bcd-a656-9d2b6a590ddb/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/53862299-39a5-46fb-ae49-2b4a19465615/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977857335688458 {"id":7364977857335688458,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Sat May 4 11:16:06 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2379923801.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public {"level":"warn","ts":"2024-05-04T11:16:07.742399+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e4a700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:07.743029+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00143cfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:07.799504+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010d41c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release public/default + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:10 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {UpstreamID:7364977857335688458 Namespace:default ID:41053777-e1a1-47f8-a8ef-ac0f4645f6b1 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:16:02.455842566 +0800 CST StartTs:449522578011717633 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e3b200 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522581485387780} {CheckpointTs:449522578496684278 MinTableBarrierTs:449522583372562443 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2528bb96-79de-4bcd-a656-9d2b6a590ddb {"id":"2528bb96-79de-4bcd-a656-9d2b6a590ddb","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792564} /tidb/cdc/default/__cdc_meta__/capture/2a5c45fd-77fe-4d28-a26a-5af9bb389d71 {"id":"2a5c45fd-77fe-4d28-a26a-5af9bb389d71","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792567} /tidb/cdc/default/__cdc_meta__/capture/53862299-39a5-46fb-ae49-2b4a19465615 {"id":"53862299-39a5-46fb-ae49-2b4a19465615","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792552} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938df2 53862299-39a5-46fb-ae49-2b4a19465615 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938e6e 2528bb96-79de-4bcd-a656-9d2b6a590ddb /tidb/cdc/default/__cdc_meta__/owner/22318f4199938e9d 2a5c45fd-77fe-4d28-a26a-5af9bb389d71 /tidb/cdc/default/default/changefeed/info/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"upstream-id":7364977857335688458,"namespace":"default","changefeed-id":"41053777-e1a1-47f8-a8ef-ac0f4645f6b1","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:16:02.455842566+08:00","start-ts":449522578011717633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522581485387780} /tidb/cdc/default/default/changefeed/status/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":449522578496684379,"min-table-barrier-ts":449522583372562443,"admin-job-type":0} /tidb/cdc/default/default/task/position/2528bb96-79de-4bcd-a656-9d2b6a590ddb/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/2a5c45fd-77fe-4d28-a26a-5af9bb389d71/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/53862299-39a5-46fb-ae49-2b4a19465615/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977857335688458 {"id":7364977857335688458,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {UpstreamID:7364977857335688458 Namespace:default ID:41053777-e1a1-47f8-a8ef-ac0f4645f6b1 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:16:02.455842566 +0800 CST StartTs:449522578011717633 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e3b200 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522581485387780} {CheckpointTs:449522578496684278 MinTableBarrierTs:449522583372562443 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2528bb96-79de-4bcd-a656-9d2b6a590ddb {"id":"2528bb96-79de-4bcd-a656-9d2b6a590ddb","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792564} /tidb/cdc/default/__cdc_meta__/capture/2a5c45fd-77fe-4d28-a26a-5af9bb389d71 {"id":"2a5c45fd-77fe-4d28-a26a-5af9bb389d71","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792567} /tidb/cdc/default/__cdc_meta__/capture/53862299-39a5-46fb-ae49-2b4a19465615 {"id":"53862299-39a5-46fb-ae49-2b4a19465615","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792552} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938df2 53862299-39a5-46fb-ae49-2b4a19465615 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938e6e 2528bb96-79de-4bcd-a656-9d2b6a590ddb /tidb/cdc/default/__cdc_meta__/owner/22318f4199938e9d 2a5c45fd-77fe-4d28-a26a-5af9bb389d71 /tidb/cdc/default/default/changefeed/info/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"upstream-id":7364977857335688458,"namespace":"default","changefeed-id":"41053777-e1a1-47f8-a8ef-ac0f4645f6b1","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:16:02.455842566+08:00","start-ts":449522578011717633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522581485387780} /tidb/cdc/default/default/changefeed/status/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":449522578496684379,"min-table-barrier-ts":449522583372562443,"admin-job-type":0} /tidb/cdc/default/default/task/position/2528bb96-79de-4bcd-a656-9d2b6a590ddb/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/2a5c45fd-77fe-4d28-a26a-5af9bb389d71/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/53862299-39a5-46fb-ae49-2b4a19465615/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977857335688458 {"id":7364977857335688458,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {UpstreamID:7364977857335688458 Namespace:default ID:41053777-e1a1-47f8-a8ef-ac0f4645f6b1 SinkURI:pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 11:16:02.455842566 +0800 CST StartTs:449522578011717633 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003e3b200 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522581485387780} {CheckpointTs:449522578496684278 MinTableBarrierTs:449522583372562443 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2528bb96-79de-4bcd-a656-9d2b6a590ddb {"id":"2528bb96-79de-4bcd-a656-9d2b6a590ddb","address":"127.0.0.1:8301","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792564} /tidb/cdc/default/__cdc_meta__/capture/2a5c45fd-77fe-4d28-a26a-5af9bb389d71 {"id":"2a5c45fd-77fe-4d28-a26a-5af9bb389d71","address":"127.0.0.1:8302","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792567} /tidb/cdc/default/__cdc_meta__/capture/53862299-39a5-46fb-ae49-2b4a19465615 {"id":"53862299-39a5-46fb-ae49-2b4a19465615","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792552} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938df2 53862299-39a5-46fb-ae49-2b4a19465615 /tidb/cdc/default/__cdc_meta__/owner/22318f4199938e6e 2528bb96-79de-4bcd-a656-9d2b6a590ddb /tidb/cdc/default/__cdc_meta__/owner/22318f4199938e9d 2a5c45fd-77fe-4d28-a26a-5af9bb389d71 /tidb/cdc/default/default/changefeed/info/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"upstream-id":7364977857335688458,"namespace":"default","changefeed-id":"41053777-e1a1-47f8-a8ef-ac0f4645f6b1","sink-uri":"pulsar://127.0.0.1:6650/ticdc-move-table-test-5231?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T11:16:02.455842566+08:00","start-ts":449522578011717633,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522581485387780} /tidb/cdc/default/default/changefeed/status/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":449522578496684379,"min-table-barrier-ts":449522583372562443,"admin-job-type":0} /tidb/cdc/default/default/task/position/2528bb96-79de-4bcd-a656-9d2b6a590ddb/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/2a5c45fd-77fe-4d28-a26a-5af9bb389d71/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/53862299-39a5-46fb-ae49-2b4a19465615/41053777-e1a1-47f8-a8ef-ac0f4645f6b1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977857335688458 {"id":7364977857335688458,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table move_table.usertable exists go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/xdg/scram v1.0.5 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 ++ date + echo '[Sat May 4 11:16:10 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:16:10 CST 2024] <<<<<< pulsar is ready >>>>>> go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 11:16:11 CST 2024] <<<<<< START Pulsar consumer in changefeed_auto_stop case >>>>>> check_changefeed_state http://127.0.0.1:2379 c96f3720-f5a6-4fa2-b257-8a737fc08b89 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=c96f3720-f5a6-4fa2-b257-8a737fc08b89 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c96f3720-f5a6-4fa2-b257-8a737fc08b89 -s + info='{ "upstream_id": 7364977893224571227, "namespace": "default", "id": "c96f3720-f5a6-4fa2-b257-8a737fc08b89", "state": "normal", "checkpoint_tso": 449522580099694595, "checkpoint_time": "2024-05-04 11:15:57.143", "error": null }' + echo '{ "upstream_id": 7364977893224571227, "namespace": "default", "id": "c96f3720-f5a6-4fa2-b257-8a737fc08b89", "state": "normal", "checkpoint_tso": 449522580099694595, "checkpoint_time": "2024-05-04 11:15:57.143", "error": null }' { "upstream_id": 7364977893224571227, "namespace": "default", "id": "c96f3720-f5a6-4fa2-b257-8a737fc08b89", "state": "normal", "checkpoint_tso": 449522580099694595, "checkpoint_time": "2024-05-04 11:15:57.143", "error": null } ++ echo '{' '"upstream_id":' 7364977893224571227, '"namespace":' '"default",' '"id":' '"c96f3720-f5a6-4fa2-b257-8a737fc08b89",' '"state":' '"normal",' '"checkpoint_tso":' 449522580099694595, '"checkpoint_time":' '"2024-05-04' '11:15:57.143",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7364977893224571227, '"namespace":' '"default",' '"id":' '"c96f3720-f5a6-4fa2-b257-8a737fc08b89",' '"state":' '"normal",' '"checkpoint_tso":' 449522580099694595, '"checkpoint_time":' '"2024-05-04' '11:15:57.143",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later go: downloading github.com/ardielle/ardielle-go v1.5.2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0{"level":"warn","ts":"2024-05-04T11:16:13.743024+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e4a700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:13.744954+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00143cfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:13.800493+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010d41c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_2.usertable not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0668d87c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:16351, start at 2024-05-04 11:16:16.577825482 +0800 CST m=+5.935456932 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:16.584 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:16.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:16.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0668d87c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:16351, start at 2024-05-04 11:16:16.577825482 +0800 CST m=+5.935456932 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:16.584 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:16.543 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:16.543 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0668dbf8000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:16437, start at 2024-05-04 11:16:16.777501562 +0800 CST m=+6.077120230 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:16.784 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:16.766 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:16.766 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table changefeed_auto_stop_3.usertable not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0{"level":"warn","ts":"2024-05-04T11:16:19.744513+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e4a700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:19.745541+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00143cfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:19.801693+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010d41c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.17758.out cli tso query --pd=http://127.0.0.1:2379 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-05-04T11:16:21.730569+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e4a700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-04T11:16:21.730626+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-04T11:16:21.732711+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00143cfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-04T11:16:21.732766+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-04T11:16:21.789939+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010d41c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-04T11:16:21.790001+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449522586443841537 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522586443841537 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 11:16:22 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1779317795.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fa940003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10695, start at 2024-05-04 11:15:45.958552213 +0800 CST m=+5.260093566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fa940003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10695, start at 2024-05-04 11:15:45.958552213 +0800 CST m=+5.260093566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fb440013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10786, start at 2024-05-04 11:15:46.031079689 +0800 CST m=+5.273318153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:46.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:46.001 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:46.001 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later [2024/05/04 11:16:23.351 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/04 11:16:23.354 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/04 11:16:23.355 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=53862299-39a5-46fb-ae49-2b4a19465615] [2024/05/04 11:16:23.355 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/04 11:16:23.355 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/04 11:16:23.511 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=41053777-e1a1-47f8-a8ef-ac0f4645f6b1] [captureID=2528bb96-79de-4bcd-a656-9d2b6a590ddb] [processorDetail="{\"table_ids\":[]}"] [2024/05/04 11:16:23.711 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=41053777-e1a1-47f8-a8ef-ac0f4645f6b1] [captureID=2a5c45fd-77fe-4d28-a26a-5af9bb389d71] [processorDetail="{\"table_ids\":[]}"] [2024/05/04 11:16:23.911 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=41053777-e1a1-47f8-a8ef-ac0f4645f6b1] [captureID=53862299-39a5-46fb-ae49-2b4a19465615] [processorDetail="{\"table_ids\":[106]}"] [2024/05/04 11:16:23.911 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"2528bb96-79de-4bcd-a656-9d2b6a590ddb\":[],\"2a5c45fd-77fe-4d28-a26a-5af9bb389d71\":[],\"53862299-39a5-46fb-ae49-2b4a19465615\":[{\"ID\":106,\"Changefeed\":\"41053777-e1a1-47f8-a8ef-ac0f4645f6b1\"}]}"] [2024/05/04 11:16:23.911 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=41053777-e1a1-47f8-a8ef-ac0f4645f6b1&target-cp-id=2a5c45fd-77fe-4d28-a26a-5af9bb389d71&table-id=106"] [2024/05/04 11:16:23.961 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/05/04 11:16:23.961 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=53862299-39a5-46fb-ae49-2b4a19465615] [targetCapture=2a5c45fd-77fe-4d28-a26a-5af9bb389d71] table move_table.check1 not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0{"level":"warn","ts":"2024-05-04T11:16:25.74607+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e4a700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:25.74663+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00143cfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:25.802678+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010d41c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:25 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact table move_table.check1 not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:27 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae5314be-5d00-437b-b47d-273ddd1fecd8 {"id":"ae5314be-5d00-437b-b47d-273ddd1fecd8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a0f09d2 ae5314be-5d00-437b-b47d-273ddd1fecd8 /tidb/cdc/default/default/upstream/7364977997631196837 {"id":7364977997631196837,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae5314be-5d00-437b-b47d-273ddd1fecd8 {"id":"ae5314be-5d00-437b-b47d-273ddd1fecd8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a0f09d2 ae5314be-5d00-437b-b47d-273ddd1fecd8 /tidb/cdc/default/default/upstream/7364977997631196837 {"id":7364977997631196837,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ae5314be-5d00-437b-b47d-273ddd1fecd8 {"id":"ae5314be-5d00-437b-b47d-273ddd1fecd8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792583} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a0f09d2 ae5314be-5d00-437b-b47d-273ddd1fecd8 /tidb/cdc/default/default/upstream/7364977997631196837 {"id":7364977997631196837,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:16:27 CST 2024] <<<<<< START pulsar cluster in savepoint case >>>>>>' [Sat May 4 11:16:27 CST 2024] <<<<<< START pulsar cluster in savepoint case >>>>>> + workdir=/tmp/tidb_cdc_test/savepoint + cluster_type=normal + cd /tmp/tidb_cdc_test/savepoint + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/savepoint/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/savepoint/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/savepoint/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/savepoint/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/savepoint/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/savepoint/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/savepoint/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/savepoint/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/savepoint/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/savepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/savepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/savepoint/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:16:27 CST 2024] <<<<<< START pulsar cluster in normal mode in savepoint case >>>>>>' [Sat May 4 11:16:27 CST 2024] <<<<<< START pulsar cluster in normal mode in savepoint case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/savepoint/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/savepoint/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/savepoint/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table move_table.check1 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066981780005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2, pid:9301, start at 2024-05-04 11:16:27.362975832 +0800 CST m=+5.104684661 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:27.369 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:27.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:27.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066981780005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2, pid:9301, start at 2024-05-04 11:16:27.362975832 +0800 CST m=+5.104684661 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:27.369 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:27.358 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:27.358 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d06698480000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-rl3x-46rh2, pid:9378, start at 2024-05-04 11:16:27.564479037 +0800 CST m=+5.256406606 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:18:27.573 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:16:27.552 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:06:27.552 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table move_table.check1 not exists for 4-th check, retry later [Sat May 4 11:16:30 CST 2024] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.1069610698.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0{"level":"warn","ts":"2024-05-04T11:16:31.747191+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00143cfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:31.748071+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e4a700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:31.804261+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010d41c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1714792592.1726394,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0034f2380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714792592.1726816,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714792592.2356875,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00237a000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714792592.2357218,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 table move_table.check1 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":1714792592.8778567,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022628c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714792592.877913,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:16:33 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1088b393-64a5-413f-8f17-bd02230fd99d {"id":"1088b393-64a5-413f-8f17-bd02230fd99d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a3a74c5 1088b393-64a5-413f-8f17-bd02230fd99d /tidb/cdc/default/default/upstream/7364978047440684722 {"id":7364978047440684722,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1088b393-64a5-413f-8f17-bd02230fd99d {"id":"1088b393-64a5-413f-8f17-bd02230fd99d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a3a74c5 1088b393-64a5-413f-8f17-bd02230fd99d /tidb/cdc/default/default/upstream/7364978047440684722 {"id":7364978047440684722,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1088b393-64a5-413f-8f17-bd02230fd99d {"id":"1088b393-64a5-413f-8f17-bd02230fd99d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792590} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419a3a74c5 1088b393-64a5-413f-8f17-bd02230fd99d /tidb/cdc/default/default/upstream/7364978047440684722 {"id":7364978047440684722,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 11:16:33 CST 2024] <<<<<< START Pulsar consumer in sink_hang case >>>>>> + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table move_table.check1 not exists for 6-th check, retry later check_changefeed_status 127.0.0.1:8300 ac1916a9-53ea-45dc-92c4-e432b316f34f normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=ac1916a9-53ea-45dc-92c4-e432b316f34f + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/ac1916a9-53ea-45dc-92c4-e432b316f34f/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1251 0 --:--:-- --:--:-- --:--:-- 1246 100 86 100 86 0 0 1249 0 --:--:-- --:--:-- --:--:-- 1246 + info='{"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589737418761}' + echo '{"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589737418761}' {"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589737418761} ++ echo '{"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589737418761}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589737418761}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 ac1916a9-53ea-45dc-92c4-e432b316f34f normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=ac1916a9-53ea-45dc-92c4-e432b316f34f + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/ac1916a9-53ea-45dc-92c4-e432b316f34f/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 1093 0 --:--:-- --:--:-- --:--:-- 1102 + info='{"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589868228641}' + echo '{"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589868228641}' {"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589868228641} ++ echo '{"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589868228641}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":449522589868228641,"checkpoint_ts":449522589868228641}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 exists table sink_hang.t2 not exists for 1-th check, retry later public/default table move_table.check1 not exists for 7-th check, retry later ++ date + echo '[Sat May 4 11:16:36 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 11:16:36 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.18270.out cli changefeed create --start-ts=449522586443841537 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-savepoint-test-922?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: b152c3d2-03c8-485c-a83a-d06fcc3725e2 Info: {"upstream_id":7364977997631196837,"namespace":"default","id":"b152c3d2-03c8-485c-a83a-d06fcc3725e2","sink_uri":"pulsar://127.0.0.1:6650/ticdc-savepoint-test-922?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T11:16:37.506860361+08:00","start_ts":449522586443841537,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522586443841537,"checkpoint_ts":449522586443841537,"checkpoint_time":"2024-05-04 11:16:21.344"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0{"level":"warn","ts":"2024-05-04T11:16:37.748947+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e4a700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:37.749333+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00143cfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) table sink_hang.t2 exists [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } {"level":"warn","ts":"2024-05-04T11:16:37.805905+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010d41c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} [Pipeline] // timeout [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later [Pipeline] // stage [Pipeline] } table move_table.check1 not exists for 8-th check, retry later [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x [Sat May 4 11:16:39 CST 2024] <<<<<< START Pulsar consumer in savepoint case >>>>>> table savepoint.finish_mark not exists for 1-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table move_table.check1 not exists for 9-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fa940003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10695, start at 2024-05-04 11:15:45.958552213 +0800 CST m=+5.260093566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fa940003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10695, start at 2024-05-04 11:15:45.958552213 +0800 CST m=+5.260093566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fb440013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10786, start at 2024-05-04 11:15:46.031079689 +0800 CST m=+5.273318153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:46.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:46.001 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:46.001 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table savepoint.finish_mark not exists for 2-th check, retry later ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... check diff failed 3-th time, retry later table move_table.check1 exists check diff successfully 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test table savepoint.finish_mark exists check diff successfully {"level":"warn","ts":"2024-05-04T11:16:43.750358+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000e4a700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:43.750367+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00143cfc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-04T11:16:43.806528+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010d41c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... check diff failed 4-th time, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:45 CST 2024] <<<<<< run test case savepoint success! >>>>>> [2024/05/04 11:16:45.746 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/04 11:16:45.748 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/04 11:16:45.748 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=53862299-39a5-46fb-ae49-2b4a19465615] [2024/05/04 11:16:45.749 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/04 11:16:45.749 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/04 11:16:45.911 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=41053777-e1a1-47f8-a8ef-ac0f4645f6b1] [captureID=2528bb96-79de-4bcd-a656-9d2b6a590ddb] [processorDetail="{\"table_ids\":[]}"] [2024/05/04 11:16:46.111 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=41053777-e1a1-47f8-a8ef-ac0f4645f6b1] [captureID=2a5c45fd-77fe-4d28-a26a-5af9bb389d71] [processorDetail="{\"table_ids\":[]}"] [2024/05/04 11:16:46.311 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=41053777-e1a1-47f8-a8ef-ac0f4645f6b1] [captureID=53862299-39a5-46fb-ae49-2b4a19465615] [processorDetail="{\"table_ids\":[108,110]}"] [2024/05/04 11:16:46.311 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"2528bb96-79de-4bcd-a656-9d2b6a590ddb\":[],\"2a5c45fd-77fe-4d28-a26a-5af9bb389d71\":[],\"53862299-39a5-46fb-ae49-2b4a19465615\":[{\"ID\":108,\"Changefeed\":\"41053777-e1a1-47f8-a8ef-ac0f4645f6b1\"},{\"ID\":110,\"Changefeed\":\"41053777-e1a1-47f8-a8ef-ac0f4645f6b1\"}]}"] [2024/05/04 11:16:46.311 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=41053777-e1a1-47f8-a8ef-ac0f4645f6b1&target-cp-id=2a5c45fd-77fe-4d28-a26a-5af9bb389d71&table-id=108"] [2024/05/04 11:16:46.361 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/04 11:16:46.361 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=41053777-e1a1-47f8-a8ef-ac0f4645f6b1&target-cp-id=2a5c45fd-77fe-4d28-a26a-5af9bb389d71&table-id=110"] check diff failed 5-th time, retry later [2024/05/04 11:16:46.411 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=110] [2024/05/04 11:16:46.411 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=53862299-39a5-46fb-ae49-2b4a19465615] [targetCapture=2a5c45fd-77fe-4d28-a26a-5af9bb389d71] check diff successfully table move_table.check2 not exists for 1-th check, retry later check diff failed 6-th time, retry later table move_table.check2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 11:16:50 CST 2024] <<<<<< run test case move_table success! >>>>>> check diff failed 7-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 8-th time, retry later check diff failed 9-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... Verifying downstream PD is started... check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 11:16:58 CST 2024] <<<<<< run test case sink_hang success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fa940003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10695, start at 2024-05-04 11:15:45.958552213 +0800 CST m=+5.260093566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fa940003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10695, start at 2024-05-04 11:15:45.958552213 +0800 CST m=+5.260093566 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:45.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:45.957 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:45.957 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0666fb440013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-xns4-1gpvr, pid:10786, start at 2024-05-04 11:15:46.031079689 +0800 CST m=+5.273318153 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:17:46.037 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:15:46.001 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:05:46.001 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.16385.out cli tso query --pd=http://127.0.0.1:2379 start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x + tso='449522597055954947 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522597055954947 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449522597055954947 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Sat May 4 11:17:03 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1642416426.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:06 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7364977868863157362 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-04 11:15:56.411058083 +0800 CST StartTs:449522578596036609 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f2b440 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522579906756609} {CheckpointTs:449522580260913156 MinTableBarrierTs:449522580260913156 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6dc3b17b-fb7d-4fd9-975b-2e07c7adf847 {"id":"6dc3b17b-fb7d-4fd9-975b-2e07c7adf847","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792623} /tidb/cdc/default/__cdc_meta__/capture/ac7c66b5-acd1-415b-92b1-64a89a224b8a {"id":"ac7c66b5-acd1-415b-92b1-64a89a224b8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199924ad6 ac7c66b5-acd1-415b-92b1-64a89a224b8a /tidb/cdc/default/__cdc_meta__/owner/22318f419abd5ed5 6dc3b17b-fb7d-4fd9-975b-2e07c7adf847 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7364977868863157362,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T11:15:56.411058083+08:00","start-ts":449522578596036609,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522579906756609} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449522580260913156,"min-table-barrier-ts":449522580260913156,"admin-job-type":0} /tidb/cdc/default/default/task/position/6dc3b17b-fb7d-4fd9-975b-2e07c7adf847/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ac7c66b5-acd1-415b-92b1-64a89a224b8a/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977868863157362 {"id":7364977868863157362,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7364977868863157362 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-04 11:15:56.411058083 +0800 CST StartTs:449522578596036609 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f2b440 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522579906756609} {CheckpointTs:449522580260913156 MinTableBarrierTs:449522580260913156 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6dc3b17b-fb7d-4fd9-975b-2e07c7adf847 {"id":"6dc3b17b-fb7d-4fd9-975b-2e07c7adf847","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792623} /tidb/cdc/default/__cdc_meta__/capture/ac7c66b5-acd1-415b-92b1-64a89a224b8a {"id":"ac7c66b5-acd1-415b-92b1-64a89a224b8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199924ad6 ac7c66b5-acd1-415b-92b1-64a89a224b8a /tidb/cdc/default/__cdc_meta__/owner/22318f419abd5ed5 6dc3b17b-fb7d-4fd9-975b-2e07c7adf847 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7364977868863157362,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T11:15:56.411058083+08:00","start-ts":449522578596036609,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522579906756609} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449522580260913156,"min-table-barrier-ts":449522580260913156,"admin-job-type":0} /tidb/cdc/default/default/task/position/6dc3b17b-fb7d-4fd9-975b-2e07c7adf847/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ac7c66b5-acd1-415b-92b1-64a89a224b8a/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":nu+ grep -q 'failed to get info:' ll} /tidb/cdc/default/default/upstream/7364977868863157362 {"id":7364977868863157362,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7364977868863157362 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-04 11:15:56.411058083 +0800 CST StartTs:449522578596036609 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003f2b440 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-54-gb297eae37 Epoch:449522579906756609} {CheckpointTs:449522580260913156 MinTableBarrierTs:449522580260913156 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6dc3b17b-fb7d-4fd9-975b-2e07c7adf847 {"id":"6dc3b17b-fb7d-4fd9-975b-2e07c7adf847","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792623} /tidb/cdc/default/__cdc_meta__/capture/ac7c66b5-acd1-415b-92b1-64a89a224b8a {"id":"ac7c66b5-acd1-415b-92b1-64a89a224b8a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792553} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4199924ad6 ac7c66b5-acd1-415b-92b1-64a89a224b8a /tidb/cdc/default/__cdc_meta__/owner/22318f419abd5ed5 6dc3b17b-fb7d-4fd9-975b-2e07c7adf847 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7364977868863157362,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-04T11:15:56.411058083+08:00","start-ts":449522578596036609,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-54-gb297eae37","epoch":449522579906756609} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449522580260913156,"min-table-barrier-ts":449522580260913156,"admin-job-type":0} /tidb/cdc/default/default/task/position/6dc3b17b-fb7d-4fd9-975b-2e07c7adf847/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/ac7c66b5-acd1-415b-92b1-64a89a224b8a/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7364977868863157362 {"id":7364977868863157362,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449522597055954947 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.16472.out cli changefeed create --start-ts=449522597055954947 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: test-2 Info: {"upstream_id":7364977868863157362,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:17:07.783821129+08:00","start_ts":449522597055954947,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522597055954947,"checkpoint_ts":449522597055954947,"checkpoint_time":"2024-05-04 11:17:01.826"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1498/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066c3d940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:14978, start at 2024-05-04 11:17:12.212708262 +0800 CST m=+5.306456382 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:12.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:12.214 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:12.214 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066c496c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:19493, start at 2024-05-04 11:17:12.961831869 +0800 CST m=+5.332901146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:12.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:12.973 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:12.973 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066c3d940014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:14978, start at 2024-05-04 11:17:12.212708262 +0800 CST m=+5.306456382 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:12.221 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:12.214 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:12.214 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066c3e40000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-7r7s-0rndn, pid:15061, start at 2024-05-04 11:17:12.224876137 +0800 CST m=+5.263802406 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:12.232 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:12.208 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:12.208 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066c61dc0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt, pid:28279, start at 2024-05-04 11:17:14.495591108 +0800 CST m=+5.190061822 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:14.507 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:14.487 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:14.487 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066c61dc0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt, pid:28279, start at 2024-05-04 11:17:14.495591108 +0800 CST m=+5.190061822 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:14.507 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:14.487 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:14.487 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066c62880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-79hx-53hgt, pid:28353, start at 2024-05-04 11:17:14.570889945 +0800 CST m=+5.212202189 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:14.579 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:14.581 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:14.581 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066c496c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:19493, start at 2024-05-04 11:17:12.961831869 +0800 CST m=+5.332901146 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:12.969 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:12.973 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:12.973 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d066c4ae40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1498-j8s1-2gnf5, pid:19576, start at 2024-05-04 11:17:13.038639051 +0800 CST m=+5.355824104 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-11:19:13.046 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-11:17:13.017 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-11:07:13.017 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sat May 4 11:17:17 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16512.out cli tso query --pd=http://127.0.0.1:2379 + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.2978429786.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.20969.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449522601181577217 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522601181577217 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449522601181577217 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sat May 4 11:17:19 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1655016552.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449522601406758913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449522601406758913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449522601406758913 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Sat May 4 11:17:19 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2100121003.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:20 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc6eca53-27ec-4673-afa8-808e45ac5959 {"id":"dc6eca53-27ec-4673-afa8-808e45ac5959","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419aee41c6 dc6eca53-27ec-4673-afa8-808e45ac5959 /tidb/cdc/default/default/upstream/7364978244949421408 {"id":7364978244949421408,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc6eca53-27ec-4673-afa8-808e45ac5959 {"id":"dc6eca53-27ec-4673-afa8-808e45ac5959","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419aee41c6 dc6eca53-27ec-4673-afa8-808e45ac5959 /tidb/cdc/default/default/upstream/7364978244949421408 {"id":7364978244949421408,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dc6eca53-27ec-4673-afa8-808e45ac5959 {"id":"dc6eca53-27ec-4673-afa8-808e45ac5959","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792637} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419aee41c6 dc6eca53-27ec-4673-afa8-808e45ac5959 /tidb/cdc/default/default/upstream/7364978244949421408 {"id":7364978244949421408,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 11:17:20 CST 2024] <<<<<< START pulsar cluster in autorandom case >>>>>>' [Sat May 4 11:17:20 CST 2024] <<<<<< START pulsar cluster in autorandom case >>>>>> + workdir=/tmp/tidb_cdc_test/autorandom + cluster_type=normal + cd /tmp/tidb_cdc_test/autorandom + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/autorandom/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/autorandom/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/autorandom/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/autorandom/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/autorandom/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/autorandom/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/autorandom/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/autorandom/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/autorandom/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/autorandom/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/autorandom/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/autorandom/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 11:17:20 CST 2024] <<<<<< START pulsar cluster in normal mode in autorandom case >>>>>>' [Sat May 4 11:17:20 CST 2024] <<<<<< START pulsar cluster in normal mode in autorandom case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/autorandom/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/autorandom/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/autorandom/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:22 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc1b1c01-c5b5-4bf8-8652-7cb44f86e114 {"id":"cc1b1c01-c5b5-4bf8-8652-7cb44f86e114","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ae4fdd4 cc1b1c01-c5b5-4bf8-8652-7cb44f86e114 /tidb/cdc/default/default/upstream/7364978237601193222 {"id":7364978237601193222,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc1b1c01-c5b5-4bf8-8652-7cb44f86e114 {"id":"cc1b1c01-c5b5-4bf8-8652-7cb44f86e114","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ae4fdd4 cc1b1c01-c5b5-4bf8-8652-7cb44f86e114 /tidb/cdc/default/default/upstream/7364978237601193222 {"id":7364978237601193222,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc1b1c01-c5b5-4bf8-8652-7cb44f86e114 {"id":"cc1b1c01-c5b5-4bf8-8652-7cb44f86e114","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ae4fdd4 cc1b1c01-c5b5-4bf8-8652-7cb44f86e114 /tidb/cdc/default/default/upstream/7364978237601193222 {"id":7364978237601193222,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449522601181577217 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16612.out cli changefeed create --start-ts=449522601181577217 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 03:17:23 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b5dec11-b2bd-4714-8635-9f2bcba5aa0d {"id":"9b5dec11-b2bd-4714-8635-9f2bcba5aa0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792640} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ae7c2d6 9b5dec11-b2bd-4714-8635-9f2bcba5aa0d /tidb/cdc/default/default/upstream/7364978239686066015 {"id":7364978239686066015,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b5dec11-b2bd-4714-8635-9f2bcba5aa0d {"id":"9b5dec11-b2bd-4714-8635-9f2bcba5aa0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792640} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ae7c2d6 9b5dec11-b2bd-4714-8635-9f2bcba5aa0d /tidb/cdc/default/default/upstream/7364978239686066015 {"id":7364978239686066015,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9b5dec11-b2bd-4714-8635-9f2bcba5aa0d {"id":"9b5dec11-b2bd-4714-8635-9f2bcba5aa0d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-54-gb297eae37","git-hash":"b297eae37cdb06fa18bc5388b04d2564a81a0b87","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714792640} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f419ae7c2d6 9b5dec11-b2bd-4714-8635-9f2bcba5aa0d /tidb/cdc/default/default/upstream/7364978239686066015 {"id":7364978239686066015,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449522601406758913 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.21063.out cli changefeed create --start-ts=449522601406758913 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Aborted by Jenkins Admin Sending interrupt signal to process Killing processes Create changefeed successfully! ID: test-1 Info: {"upstream_id":7364978239686066015,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:17:23.4876225+08:00","start_ts":449522601406758913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522601406758913,"checkpoint_ts":449522601406758913,"checkpoint_time":"2024-05-04 11:17:18.423"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes Create changefeed successfully! ID: test-1 Info: {"upstream_id":7364978237601193222,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T11:17:24.01259564+08:00","start_ts":449522601181577217,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-54-gb297eae37","resolved_ts":449522601181577217,"checkpoint_ts":449522601181577217,"checkpoint_time":"2024-05-04 11:17:17.564"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 {"level":"warn","ts":1714792643.7756422,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0035b8a80/127.0.0.1:2479","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714792643.7773845,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00187e380/127.0.0.1:2779","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 ++ stop_tidb_cluster script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes ++ stop_tidb_cluster {"level":"warn","ts":1714792644.5840836,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002244380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 ++ stop_tidb_cluster script returned exit code 143 Killing processes kill finished with exit code 0 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G13' [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: f8ea2f52-29cf-4940-b9fc-46d38aac7ac1 Finished: ABORTED