Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_pulsar_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-4xn79-380mn --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "530f00b80b7f55e4a664e40cbfc27b83c4003730" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-4xn79" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=09c52c67-c483-4dc5-bb55-5fc9e9a2a960 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-4xn79 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-100f7bb7add7cfb003dff5768bf20348b91a6b6cef38154d1625c661ef800a11 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-4xn79 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build4113026747=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-4xn7-0vkhc bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 6002 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 43a6499cc2785258 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 25 IN A 20.205.243.166 ;; Query time: 1 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Sat May 04 13:01:32 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/model/kv.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sourcemanager/manager.go, cdc/redo/reader/reader.go, cdc/sink/dmlsink/factory/factory.go, cdc/sink/dmlsink/txn/mysql/dml.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/kafka-consumer/main.go, cmd/pulsar-consumer/main.go, cmd/storage-consumer/main.go, errors.toml, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/errors/cdc_errors.go, pkg/errors/helper.go, pkg/sink/codec/open/open_protocol_decoder.go, pkg/sink/codec/open/open_protocol_message.go, tests/integration_tests/_utils/check_sync_diff, tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml, tests/integration_tests/changefeed_dup_error_restart/conf/workload, tests/integration_tests/changefeed_dup_error_restart/run.sh, tests/integration_tests/force_replicate_table/run.sh, tests/integration_tests/open_protocol_handle_key_only/data/data.sql, tests/integration_tests/open_protocol_handle_key_only/run.sh, tests/integration_tests/run_group.sh] [Pipeline] echo diff file not matched: cdc/model/kv.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534) 203635712 bytes in 5.48 secs (37191195 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/.git/ .git HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) POST git-upload-pack (656 bytes) POST git-upload-pack (973 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master * [new ref] refs/pull/10919/head -> origin/pr/10919/head HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) ๐Ÿšง Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) ๐Ÿšง Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..7f3a787ed Fast-forward cdc/model/kv.go | 5 + cdc/model/sink.go | 38 ++- cdc/model/sink_test.go | 9 +- cdc/processor/processor.go | 21 +- cdc/processor/sinkmanager/manager.go | 5 + cdc/processor/sourcemanager/manager.go | 66 +++- cdc/redo/reader/reader.go | 21 +- cdc/sink/dmlsink/factory/factory.go | 8 +- cdc/sink/dmlsink/txn/mysql/dml.go | 25 ++ cdc/sink/dmlsink/txn/mysql/mysql.go | 89 +++--- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 2 +- cmd/kafka-consumer/main.go | 4 +- cmd/pulsar-consumer/main.go | 17 +- cmd/storage-consumer/main.go | 4 +- errors.toml | 5 + pkg/applier/redo.go | 303 +++++++++++++++++- pkg/applier/redo_test.go | 347 ++++++++++++++++++++- pkg/errors/cdc_errors.go | 4 + pkg/errors/helper.go | 19 ++ pkg/sink/codec/open/open_protocol_decoder.go | 1 + pkg/sink/codec/open/open_protocol_message.go | 3 + tests/integration_tests/_utils/check_sync_diff | 2 +- .../conf/diff_config.toml | 29 ++ .../changefeed_dup_error_restart/conf/workload | 13 + .../changefeed_dup_error_restart/run.sh | 54 ++++ .../integration_tests/force_replicate_table/run.sh | 4 +- .../open_protocol_handle_key_only/data/data.sql | 2 +- .../open_protocol_handle_key_only/run.sh | 4 +- tests/integration_tests/run_group.sh | 5 +- 29 files changed, 1004 insertions(+), 105 deletions(-) create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/workload create mode 100755 tests/integration_tests/changefeed_dup_error_restart/run.sh ๐Ÿงพ Pre-merged result: 7f3a787edb079ab2f7719b4b79f735e1697cc211 7f3a787ed f 3e3d2672b f ccb0e990e f โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache saved successfully (git/pingcap/tiflow/rev-be15534-7f3a787) 203867136 bytes in 3.30 secs (61718427 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1125 0 --:--:-- --:--:-- --:--:-- 1138 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1561 0 --:--:-- --:--:-- --:--:-- 1518 100 41 100 41 0 0 1556 0 --:--:-- --:--:-- --:--:-- 1518 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1568 0 --:--:-- --:--:-- --:--:-- 1576 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2528 0 --:--:-- --:--:-- --:--:-- 2562 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz 2024-05-04 21:02:01 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz [536570515/536570515] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-04 21:02:13 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz 2024-05-04 21:02:36 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz [919098782/919098782] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz 2024-05-04 21:02:56 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz [456057803/456057803] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-04 21:03:01 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-04 21:03:02 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-04 21:03:03 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-04 21:03:03 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-04 21:03:06 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-04 21:03:15 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 21:03 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 21:03 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 21:03 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 21:03 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 21:03 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 21:03 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 21:03 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 21:03 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 21:03 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 21:03 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-91-g7f3a787ed" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 13:03:19" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7f3a787edb079ab2f7719b4b79f735e1697cc211" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-91-g7f3a787ed"' -o bin/cdc ./cmd/cdc go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading golang.org/x/net v0.24.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/IBM/sarama v1.41.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/xdg/scram v1.0.5 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading github.com/google/uuid v1.6.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading golang.org/x/sys v0.19.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_pulsar_consumer ']' + make pulsar_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-91-g7f3a787ed" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 13:05:16" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7f3a787edb079ab2f7719b4b79f735e1697cc211" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-91-g7f3a787ed"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-91-g7f3a787ed" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 13:05:23" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7f3a787edb079ab2f7719b4b79f735e1697cc211" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-91-g7f3a787ed"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-91-g7f3a787ed" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 13:05:23" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7f3a787edb079ab2f7719b4b79f735e1697cc211" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-91-g7f3a787ed"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-91-g7f3a787ed" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 13:05:23" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7f3a787edb079ab2f7719b4b79f735e1697cc211" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-91-g7f3a787ed"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-91-g7f3a787ed" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 13:05:23" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7f3a787edb079ab2f7719b4b79f735e1697cc211" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-91-g7f3a787ed"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/rs/cors v1.7.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-91-g7f3a787ed" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 13:05:23" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7f3a787edb079ab2f7719b4b79f735e1697cc211" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-91-g7f3a787ed"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-91-g7f3a787ed" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 13:05:23" -X "github.com/pingcap/tiflow/pkg/version.GitHash=7f3a787edb079ab2f7719b4b79f735e1697cc211" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-91-g7f3a787ed"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 21:10 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 21:03 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 4 21:10 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 21:09 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 21:05 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 4 21:05 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 21:05 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 4 21:05 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-91-g7f3a787ed Git Commit Hash: 7f3a787edb079ab2f7719b4b79f735e1697cc211 Git Branch: HEAD UTC Build Time: 2024-05-04 13:05:23 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache saved successfully (binary/pingcap/tiflow/cdc-integration-pulsar-test/rev-be15534-7f3a787) 1194974720 bytes in 27.77 secs (43026493 bytes/sec) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 21:10 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 21:03 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 21:10 bin -rwxr-xr-x. 1 jenkins jenkins 220M May 4 21:10 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 21:09 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 21:05 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 4 21:05 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 21:05 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 21:10 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 4 21:10 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 21:10 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 21:10 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 21:10 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 4 21:10 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 4 21:10 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 4 21:10 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 4 21:10 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 4 21:10 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 4 21:10 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 4 21:10 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 4 21:10 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 4 21:10 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 4 21:05 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 4 21:10 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 4 21:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 4 21:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 4 21:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 4 21:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 4 21:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M May 4 21:10 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 4 21:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 21:10 share -rwxr-xr-x. 1 jenkins jenkins 32M May 4 21:10 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M May 4 21:10 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 4 21:10 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 4 21:10 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 4 21:10 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 4 21:10 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 111.60 secs (33428162 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-vq1p-pgzt0 [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-sn1z-vr8b9 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-3wtc-fdmrj Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-sn1z-vr8b9 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-sn1z8-zgrpc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "b7f74609296e6f3bd97c9d0b31a0ec213abf4ff9" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-sn1z8" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-sn1z-vr8b9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-sn1z-vr8b9" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-sn1z-vr8b9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-sn1z-vr8b9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] checkout The recommended git tool is: git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p [Pipeline] { [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-vq1p-pgzt0 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-vq1p1-1vpff --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "624695b8074fd8e2b6b6a3336d3adc836da95e73" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-vq1p1" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-vq1p-pgzt0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-vq1p-pgzt0" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-vq1p-pgzt0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-vq1p-pgzt0 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7820f435; decorates RemoteLauncher[hudson.remoting.Channel@61742e8c:JNLP4-connect connection from 10.233.68.137/10.233.68.137:52992] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3448dce5; decorates RemoteLauncher[hudson.remoting.Channel@6c30237a:JNLP4-connect connection from 10.233.67.185/10.233.67.185:34432] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg [Pipeline] { Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-3wtc-fdmrj is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-3wtcg-5sk4m --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c6c29911dd83a12c49a2fa4c7270d9f84770a983" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-3wtcg" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-3wtc-fdmrj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-3wtc-fdmrj" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-3wtc-fdmrj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-3wtc-fdmrj in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] node [Pipeline] { [Pipeline] checkout The recommended git tool is: git Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-bc1j3-6m3rm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ce2135e14559a005296b3cdc0a4f1c5671699a1c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-bc1j3" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@53120d60; decorates RemoteLauncher[hudson.remoting.Channel@1d36e3df:JNLP4-connect connection from 10.233.73.179/10.233.73.179:44394] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6d2afe5c; decorates RemoteLauncher[hudson.remoting.Channel@7049ff8e:JNLP4-connect connection from 10.233.71.61/10.233.71.61:35240] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-v4bwk-wxvpl --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "456d28d0ba604f91c3f375e8be5c7f95b37f28be" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-v4bwk" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-skmft-bf1d9 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "eb888d3a593c6c28e1e767a726235d7388402d62" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-skmft" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-30rnz-h5qw8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "fc0fda71348e87a052fd299fdcdc088f20846c6f" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-30rnz" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-mbt1x-336d8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f272bdd7bf5783fec20190673ca5f49112840635" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-mbt1x" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-11rq5-sh1zp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "350a0918fe1beaebe230d12e0a15cd4231ac24a8" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-11rq5" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_pulsar_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-jdj9r-cgf7h --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "969ccc702fe498b5bd06787c4b7e05baa82bfdb5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-jdj9r" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 18.79 secs (198508968 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] { [Pipeline] sh [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G01 Run cases: open_protocol_handle_key_only PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=5d2383b5-c3d5-44d4-a320-6ef8c4a741d4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-sn1z-vr8b9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-sn1z8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-sn1z8 pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-sn1z-vr8b9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-sn1z-vr8b9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:13:01 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] { The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@73a64d6b; decorates RemoteLauncher[hudson.remoting.Channel@68e6d324:JNLP4-connect connection from 10.233.67.254/10.233.67.254:48288] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@75001f65; decorates RemoteLauncher[hudson.remoting.Channel@1641b1ba:JNLP4-connect connection from 10.233.100.162/10.233.100.162:45968] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@88bff43; decorates RemoteLauncher[hudson.remoting.Channel@4bd6003:JNLP4-connect connection from 10.233.97.87/10.233.97.87:47538] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@660ac2f2; decorates RemoteLauncher[hudson.remoting.Channel@35a62487:JNLP4-connect connection from 10.233.88.106/10.233.88.106:43854] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@37e6b3dc; decorates RemoteLauncher[hudson.remoting.Channel@3040a51b:JNLP4-connect connection from 10.233.70.201/10.233.70.201:48562] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4b68f009; decorates RemoteLauncher[hudson.remoting.Channel@d66360e:JNLP4-connect connection from 10.233.72.163/10.233.72.163:36202] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" \033[0;36m<<< Run all test success >>>\033[0m Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-lpbc3-p2zs6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "dfac80c3e49ab52d67e8cc71ac2ff6829b7243b3" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-lpbc3" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-q7cdx-4m94r --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d220a8607f4d9e730e7b5e3f50cf02bbb8642721" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-q7cdx" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-7v1cj-vm1pq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7e7d3bd598445b21538fc36523777f1d5698fbae" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-7v1cj" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-r93qw-5mhdt Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-xhmtq-9d006 --- apiVersion: "v1" kind: "Pod" metadata: --- apiVersion: "v1" kind: "Pod" metadata: annotations: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "68f1403c3a78f151f7ce3cde5464fc8f0ad81f4c" jenkins/label-digest: "bb22db55c882ac33d0040fc11ccd800c2eece2e3" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-xhmtq" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-r93qw" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws" namespace: "jenkins-tiflow" spec: affinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" operator: "In" values: - "amd64" containers: tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" resources: limits: memory: "128Mi" cpu: "100m" tty: true value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q" - name: "JENKINS_NAME" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" - name: "JENKINS_AGENT_NAME" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws" - name: "JENKINS_NAME" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" - emptyDir: medium: "" name: "workspace-volume" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2 is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-l51s9-z6rlz --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7013e04e30e4e2783aa33dd75b53ad411af14647" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-l51s9" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-xfvtz-90wwf --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "52c1f455bd51104d3ce5fc6073a3135bf7a0e5e6" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-xfvtz" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs is provisioned from template pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-kqgnr-gd5qh --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "9256b45c7e117176c4291d8301a118bf7248ca90" jenkins/label: "pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-kqgnr" name: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:pulsar-test" name: "golang" resources: limits: memory: "32Gi" cpu: "12" requests: memory: "32Gi" cpu: "12" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 16.41 secs (227389126 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 19.24 secs (193848161 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 9.58 secs (389266107 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=d8fa1993-5989-4b35-8c99-0e9648340caa BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-vq1p-pgzt0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-vq1p1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-vq1p1 pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-vq1p-pgzt0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-vq1p-pgzt0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:13:59 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=0cecba71-3cba-4a84-8386-74a5a58c44e7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-3wtc-fdmrj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-3wtcg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-3wtcg pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-3wtc-fdmrj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-3wtc-fdmrj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:13:59 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc already exists) [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=7c16b0c0-5e01-4bbb-87ea-079492edfa16 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-bc1j3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-bc1j3 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] // cache [Pipeline] } [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] // dir [Pipeline] } [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@45ee133c; decorates RemoteLauncher[hudson.remoting.Channel@1da139fe:JNLP4-connect connection from 10.233.108.238/10.233.108.238:39434] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4f8ea175; decorates RemoteLauncher[hudson.remoting.Channel@a6ef8db:JNLP4-connect connection from 10.233.107.31/10.233.107.31:44716] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@15954ec9; decorates RemoteLauncher[hudson.remoting.Channel@5c20f241:JNLP4-connect connection from 10.233.86.77/10.233.86.77:54482] will be ignored (a typical symptom is the Git executable not being run inside a designated container) No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@196ce487; decorates RemoteLauncher[hudson.remoting.Channel@5dbfc2e4:JNLP4-connect connection from 10.233.123.153/10.233.123.153:52656] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@1761a670; decorates RemoteLauncher[hudson.remoting.Channel@3e6a5972:JNLP4-connect connection from 10.233.73.76/10.233.73.76:53976] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@69a936c5; decorates RemoteLauncher[hudson.remoting.Channel@eebeec0:JNLP4-connect connection from 10.233.90.216/10.233.90.216:44608] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2b9a8880; decorates RemoteLauncher[hudson.remoting.Channel@44655af6:JNLP4-connect connection from 10.233.105.129/10.233.105.129:48130] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@16b61676; decorates RemoteLauncher[hudson.remoting.Channel@4546171f:JNLP4-connect connection from 10.233.71.131/10.233.71.131:60602] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:02 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: pulsar... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC4B6288B626B3 < X-Xss-Protection: 1; mode=block < Date: Sat, 04 May 2024 13:14:04 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Sat May 4 21:14:04 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Verifying downstream PD is started... Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: pulsar... <<================= kafka downstream isn't support syncpoint record [Sat May 4 21:14:05 CST 2024] <<<<<< run test case syncpoint success! >>>>>> Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:07 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:08 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:10 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:12 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:14 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0ef692a00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk, pid:1350, start at 2024-05-04 21:14:13.068291217 +0800 CST m=+5.163915074 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:16:13.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:14:13.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:04:13.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0ef692a00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk, pid:1350, start at 2024-05-04 21:14:13.068291217 +0800 CST m=+5.163915074 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:16:13.077 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:14:13.082 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:04:13.082 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0ef692880018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-bc1j-p0czk, pid:1436, start at 2024-05-04 21:14:13.076057164 +0800 CST m=+5.115887753 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:16:13.084 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:14:13.076 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:04:13.076 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:15 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2821.out cli tso query --pd=http://127.0.0.1:2379 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: pulsar... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend + set +x + tso='449531990909386753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449531990909386753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 21:14:17 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.28562858.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:18 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC4B65E22FCA57 < X-Xss-Protection: 1; mode=block < Date: Sat, 04 May 2024 13:14:19 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Sat May 4 21:14:19 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:14:21 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11355d3e-0868-4662-83fc-13066bb6f4ea {"id":"11355d3e-0868-4662-83fc-13066bb6f4ea","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828458} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bd7e8cd0 11355d3e-0868-4662-83fc-13066bb6f4ea /tidb/cdc/default/default/upstream/7365132085058834687 {"id":7365132085058834687,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11355d3e-0868-4662-83fc-13066bb6f4ea {"id":"11355d3e-0868-4662-83fc-13066bb6f4ea","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828458} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bd7e8cd0 11355d3e-0868-4662-83fc-13066bb6f4ea /tidb/cdc/default/default/upstream/7365132085058834687 {"id":7365132085058834687,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11355d3e-0868-4662-83fc-13066bb6f4ea {"id":"11355d3e-0868-4662-83fc-13066bb6f4ea","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828458} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bd7e8cd0 11355d3e-0868-4662-83fc-13066bb6f4ea /tidb/cdc/default/default/upstream/7365132085058834687 {"id":7365132085058834687,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:14:21 CST 2024] <<<<<< START pulsar cluster in row_format case >>>>>>' [Sat May 4 21:14:21 CST 2024] <<<<<< START pulsar cluster in row_format case >>>>>> + workdir=/tmp/tidb_cdc_test/row_format + cluster_type=normal + cd /tmp/tidb_cdc_test/row_format + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/row_format/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/row_format/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/row_format/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/row_format/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/row_format/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/row_format/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/row_format/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/row_format/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/row_format/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/row_format/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:14:21 CST 2024] <<<<<< START pulsar cluster in normal mode in row_format case >>>>>>' [Sat May 4 21:14:21 CST 2024] <<<<<< START pulsar cluster in normal mode in row_format case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/row_format/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/row_format/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/row_format/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:21 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:22 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:25 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:25 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> public/default Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 22.68 secs (164508447 bytes/sec) [Pipeline] { [Pipeline] cache ++ date + echo '[Sat May 4 21:14:27 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:14:27 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.3324.out cli changefeed create --start-ts=449531990909386753 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-row-format-test-30122?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 0a6e7315-8e54-4f0d-95f3-b0e8b29faf83 Info: {"upstream_id":7365132085058834687,"namespace":"default","id":"0a6e7315-8e54-4f0d-95f3-b0e8b29faf83","sink_uri":"pulsar://127.0.0.1:6650/ticdc-row-format-test-30122?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:14:28.304295001+08:00","start_ts":449531990909386753,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449531990909386753,"checkpoint_ts":449531990909386753,"checkpoint_time":"2024-05-04 21:14:16.533"} PASS find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:28 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> coverage: 2.4% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:28 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> + set +x [Sat May 4 21:14:29 CST 2024] <<<<<< START Pulsar consumer in row_format case >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:31 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:31 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: pulsar... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=pulsar + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run pulsar + '[' pulsar '!=' storage ']' + return + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Sat May 4 21:14:34 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Sat May 4 21:14:34 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:34 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:37 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:37 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 8.33 secs (448028383 bytes/sec) [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 1-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:40 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:40 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> table row_format.finish_mark not exists for 2-th check, retry later table row_format.finish_mark not exists for 3-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:43 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: pulsar... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:44 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> table row_format.finish_mark not exists for 4-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m table row_format.finish_mark not exists for 5-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:14:47 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> table row_format.finish_mark not exists for 6-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m table row_format.finish_mark not exists for 7-th check, retry later table row_format.finish_mark not exists for 8-th check, retry later table row_format.finish_mark not exists for 9-th check, retry later table row_format.finish_mark not exists for 10-th check, retry later table row_format.finish_mark not exists for 11-th check, retry later table row_format.finish_mark not exists for 12-th check, retry later table row_format.finish_mark not exists for 13-th check, retry later table row_format.finish_mark not exists for 14-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 25.57 secs (145907686 bytes/sec) [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 15-th check, retry later table row_format.finish_mark not exists for 16-th check, retry later table row_format.finish_mark not exists for 17-th check, retry later table row_format.finish_mark not exists for 18-th check, retry later table row_format.finish_mark not exists for 19-th check, retry later table row_format.finish_mark not exists for 20-th check, retry later table row_format.finish_mark not exists for 21-th check, retry later table row_format.finish_mark not exists for 22-th check, retry later table row_format.finish_mark not exists for 23-th check, retry later table row_format.finish_mark not exists for 24-th check, retry later table row_format.finish_mark not exists for 25-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 20.52 secs (181755916 bytes/sec) [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 26-th check, retry later table row_format.finish_mark not exists for 27-th check, retry later table row_format.finish_mark not exists for 28-th check, retry later table row_format.finish_mark not exists for 29-th check, retry later table row_format.finish_mark not exists for 30-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 5.78 secs (645691266 bytes/sec) [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 31-th check, retry later table row_format.finish_mark not exists for 32-th check, retry later table row_format.finish_mark not exists for 33-th check, retry later table row_format.finish_mark not exists for 34-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 6.31 secs (591474028 bytes/sec) [Pipeline] { [Pipeline] // timeout [Pipeline] } [Pipeline] sh [Pipeline] sh table row_format.finish_mark not exists for 35-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=2b0f79ff-8948-4097-8364-7365caadd3cb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-v4bwk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-v4bwk GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=fe64caf9-79f6-41e0-bb20-9874fd3ec3cd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-skmft GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-skmft pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=9f8744ec-0e6c-4f0f-8901-fed182656f1f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-30rnz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-30rnz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=b5f331b8-c433-4e29-90e7-215f00c761ba BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-mbt1x GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-mbt1x pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=3f3b7536-80ba-4806-b6e7-3944b95cb297 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-11rq5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8 pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-11rq5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table row_format.finish_mark not exists for 36-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=6c1fca9e-49fe-4e51-b4e0-56dbf142ebb4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-jdj9r GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-jdj9r GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc already exists) [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc already exists) table row_format.finish_mark not exists for 37-th check, retry later [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] // stage start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] } [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } Verifying downstream PD is started... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) table row_format.finish_mark not exists for 38-th check, retry later [Pipeline] // container [Pipeline] } [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow [Pipeline] { [Pipeline] // withEnv [Pipeline] } [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table row_format.finish_mark not exists for 39-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 40-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table row_format.finish_mark not exists for 41-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 42-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd37540012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p, pid:1308, start at 2024-05-04 21:16:01.895677823 +0800 CST m=+5.426412646 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:01.903 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:01.877 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:01.877 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd344c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg, pid:1293, start at 2024-05-04 21:16:01.691585029 +0800 CST m=+5.216159803 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:01.700 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:01.683 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:01.683 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd34c80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm, pid:1297, start at 2024-05-04 21:16:01.743077232 +0800 CST m=+5.137322147 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:01.749 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:01.714 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:01.714 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd34c80016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm, pid:1297, start at 2024-05-04 21:16:01.743077232 +0800 CST m=+5.137322147 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:01.749 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:01.714 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:01.714 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd363c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm, pid:1383, start at 2024-05-04 21:16:01.841109767 +0800 CST m=+5.182154340 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:01.849 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:01.807 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:01.807 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table row_format.finish_mark not exists for 43-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd4c500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8, pid:1368, start at 2024-05-04 21:16:03.264465858 +0800 CST m=+5.152480895 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:03.271 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:03.271 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:03.271 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd4c500015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8, pid:1368, start at 2024-05-04 21:16:03.264465858 +0800 CST m=+5.152480895 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:03.271 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:03.271 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:03.271 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd4dc80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8, pid:1450, start at 2024-05-04 21:16:03.353614931 +0800 CST m=+5.186812960 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:03.360 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:03.364 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:03.364 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd37540012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p, pid:1308, start at 2024-05-04 21:16:01.895677823 +0800 CST m=+5.426412646 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:01.903 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:01.877 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:01.877 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd37fc000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p, pid:1390, start at 2024-05-04 21:16:01.929175092 +0800 CST m=+5.403332793 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:01.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:01.919 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:01.919 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd344c0009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg, pid:1293, start at 2024-05-04 21:16:01.691585029 +0800 CST m=+5.216159803 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:01.700 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:01.683 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:01.683 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd35f40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg, pid:1376, start at 2024-05-04 21:16:01.805283943 +0800 CST m=+5.275719122 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:01.811 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:01.789 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:01.789 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd4f140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q, pid:1300, start at 2024-05-04 21:16:03.440363857 +0800 CST m=+5.099029831 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:03.449 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:03.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:03.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd4f140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q, pid:1300, start at 2024-05-04 21:16:03.440363857 +0800 CST m=+5.099029831 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:03.449 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:03.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:03.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd5224000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q, pid:1394, start at 2024-05-04 21:16:03.602644678 +0800 CST m=+5.205390597 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:03.609 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:03.593 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:03.593 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sat May 4 21:16:04 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.28652867.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 44-th check, retry later ++ date + echo '[Sat May 4 21:16:06 CST 2024] <<<<<< START pulsar cluster in gc_safepoint case >>>>>>' [Sat May 4 21:16:06 CST 2024] <<<<<< START pulsar cluster in gc_safepoint case >>>>>> + workdir=/tmp/tidb_cdc_test/gc_safepoint + cluster_type=normal + cd /tmp/tidb_cdc_test/gc_safepoint + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/gc_safepoint/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/gc_safepoint/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/gc_safepoint/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/gc_safepoint/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/gc_safepoint/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/gc_safepoint/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/gc_safepoint/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/gc_safepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/gc_safepoint/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/gc_safepoint/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:16:06 CST 2024] <<<<<< START pulsar cluster in normal mode in gc_safepoint case >>>>>>' [Sat May 4 21:16:06 CST 2024] <<<<<< START pulsar cluster in normal mode in gc_safepoint case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/gc_safepoint/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/gc_safepoint/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/gc_safepoint/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ++ date + echo '[Sat May 4 21:16:06 CST 2024] <<<<<< START pulsar cluster in processor_err_chan case >>>>>>' [Sat May 4 21:16:06 CST 2024] <<<<<< START pulsar cluster in processor_err_chan case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_err_chan + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_err_chan + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_err_chan/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_err_chan/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_err_chan/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_err_chan/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_err_chan/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_err_chan/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_err_chan/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_err_chan/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_err_chan/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_err_chan/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:16:06 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_err_chan case >>>>>>' [Sat May 4 21:16:06 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_err_chan case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_err_chan/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_err_chan/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_err_chan/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2829.out cli tso query --pd=http://127.0.0.1:2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2826.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd77c00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p, pid:1409, start at 2024-05-04 21:16:06.023222929 +0800 CST m=+5.223827698 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:06.032 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:06.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:06.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd77c00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p, pid:1409, start at 2024-05-04 21:16:06.023222929 +0800 CST m=+5.223827698 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:06.032 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:06.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:06.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0efd79740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p, pid:1494, start at 2024-05-04 21:16:06.140372914 +0800 CST m=+5.282248610 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:06.149 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:06.109 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:06.109 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table row_format.finish_mark not exists for 45-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:16:08 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/514f32c6-6664-4e7b-a444-999c90fef1ec {"id":"514f32c6-6664-4e7b-a444-999c90fef1ec","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828565} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf2362c3 514f32c6-6664-4e7b-a444-999c90fef1ec /tidb/cdc/default/default/upstream/7365132554912501842 {"id":7365132554912501842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/514f32c6-6664-4e7b-a444-999c90fef1ec {"id":"514f32c6-6664-4e7b-a444-999c90fef1ec","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828565} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf2362c3 514f32c6-6664-4e7b-a444-999c90fef1ec /tidb/cdc/default/default/upstream/7365132554912501842 {"id":7365132554912501842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/514f32c6-6664-4e7b-a444-999c90fef1ec {"id":"514f32c6-6664-4e7b-a444-999c90fef1ec","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828565} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf2362c3 514f32c6-6664-4e7b-a444-999c90fef1ec /tidb/cdc/default/default/upstream/7365132554912501842 {"id":7365132554912501842,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:16:08 CST 2024] <<<<<< START pulsar cluster in charset_gbk case >>>>>>' [Sat May 4 21:16:08 CST 2024] <<<<<< START pulsar cluster in charset_gbk case >>>>>> + workdir=/tmp/tidb_cdc_test/charset_gbk + cluster_type=mtls + cd /tmp/tidb_cdc_test/charset_gbk + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/charset_gbk/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/charset_gbk/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/charset_gbk/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/charset_gbk/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/charset_gbk/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus ....................................................................................................+++ ......................+++ e is 65537 (0x10001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus ......................................................................................+++ ...+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 Signature ok subject=/CN=server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus .......+++ .....................+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=/CN=broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/charset_gbk/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/charset_gbk/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/charset_gbk/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/charset_gbk/ca.cert.pem' ++ date + echo '[Sat May 4 21:16:08 CST 2024] <<<<<< START pulsar cluster in mtls mode in charset_gbk case >>>>>>' [Sat May 4 21:16:08 CST 2024] <<<<<< START pulsar cluster in mtls mode in charset_gbk case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/charset_gbk/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/charset_gbk/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/charset_gbk/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + set +x + tso='449532019901202434 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532019901202434 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 21:16:08 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.28602862.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + set +x + tso='449532019902251009 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532019902251009 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "threadcount"="2" "readallfields"="true" "updateproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.user"="root" "readproportion"="0" "requestdistribution"="uniform" "operationcount"="0" "insertproportion"="0" "dotransactions"="false" "mysql.db"="sink_retry" "workload"="core" "recordcount"="10" "mysql.port"="4000" ********************************************** Run finished, takes 23.108708ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1372.8, Avg(us): 4540, Min(us): 1240, Max(us): 15740, 95th(us): 16000, 99th(us): 16000 [Sat May 4 21:16:08 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.28832885.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ++ date + echo '[Sat May 4 21:16:09 CST 2024] <<<<<< START pulsar cluster in kv_client_stream_reconnect case >>>>>>' [Sat May 4 21:16:09 CST 2024] <<<<<< START pulsar cluster in kv_client_stream_reconnect case >>>>>> + workdir=/tmp/tidb_cdc_test/kv_client_stream_reconnect + cluster_type=oauth + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/kv_client_stream_reconnect/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/kv_client_stream_reconnect/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_standalone.conf + pulsar_port=6650 + '[' oauth == mtls ']' + '[' oauth == oauth ']' + echo 'Waiting for oauth2 server to be ready...' Waiting for oauth2 server to be ready... + i=0 + nc -z localhost 9096 + oauth2-server + gen_oauth_config + echo ' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/kv_client_stream_reconnect/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ date + echo '[Sat May 4 21:16:09 CST 2024] <<<<<< START pulsar cluster in oauth mode in kv_client_stream_reconnect case >>>>>>' [Sat May 4 21:16:09 CST 2024] <<<<<< START pulsar cluster in oauth mode in kv_client_stream_reconnect case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table row_format.finish_mark not exists for 46-th check, retry later + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table row_format.finish_mark not exists for 47-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:16:11 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bff8b516-319c-42fb-95ac-5df34bea9013 {"id":"bff8b516-319c-42fb-95ac-5df34bea9013","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828569} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf26bbd8 bff8b516-319c-42fb-95ac-5df34bea9013 /tidb/cdc/default/default/upstream/7365132555989855257 {"id":7365132555989855257,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bff8b516-319c-42fb-95ac-5df34bea9013 {"id":"bff8b516-319c-42fb-95ac-5df34bea9013","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828569} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf26bbd8 bff8b516-319c-42fb-95ac-5df34bea9013 /tidb/cdc/default/default/upstream/7365132555989855257 {"id":7365132555989855257,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bff8b516-319c-42fb-95ac-5df34bea9013 {"id":"bff8b516-319c-42fb-95ac-5df34bea9013","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828569} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf26bbd8 bff8b516-319c-42fb-95ac-5df34bea9013 /tidb/cdc/default/default/upstream/7365132555989855257 {"id":7365132555989855257,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:16:11 CST 2024] <<<<<< START pulsar cluster in foreign_key case >>>>>>' [Sat May 4 21:16:11 CST 2024] <<<<<< START pulsar cluster in foreign_key case >>>>>> + workdir=/tmp/tidb_cdc_test/foreign_key + cluster_type=normal + cd /tmp/tidb_cdc_test/foreign_key + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/foreign_key/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/foreign_key/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/foreign_key/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/foreign_key/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/foreign_key/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/foreign_key/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/foreign_key/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/foreign_key/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/foreign_key/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/foreign_key/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:16:11 CST 2024] <<<<<< START pulsar cluster in normal mode in foreign_key case >>>>>>' [Sat May 4 21:16:11 CST 2024] <<<<<< START pulsar cluster in normal mode in foreign_key case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/foreign_key/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/foreign_key/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/foreign_key/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:16:11 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6c38595-dbe4-4157-80f4-96efb829faaa {"id":"f6c38595-dbe4-4157-80f4-96efb829faaa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828569} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf24a0f2 f6c38595-dbe4-4157-80f4-96efb829faaa /tidb/cdc/default/default/upstream/7365132547131950924 {"id":7365132547131950924,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6c38595-dbe4-4157-80f4-96efb829faaa {"id":"f6c38595-dbe4-4157-80f4-96efb829faaa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828569} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf24a0f2 f6c38595-dbe4-4157-80f4-96efb829faaa /tidb/cdc/default/default/upstream/7365132547131950924 {"id":7365132547131950924,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f6c38595-dbe4-4157-80f4-96efb829faaa {"id":"f6c38595-dbe4-4157-80f4-96efb829faaa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828569} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf24a0f2 f6c38595-dbe4-4157-80f4-96efb829faaa /tidb/cdc/default/default/upstream/7365132547131950924 {"id":7365132547131950924,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:16:11 CST 2024] <<<<<< START pulsar cluster in sink_retry case >>>>>>' [Sat May 4 21:16:11 CST 2024] <<<<<< START pulsar cluster in sink_retry case >>>>>> + workdir=/tmp/tidb_cdc_test/sink_retry + cluster_type=normal + cd /tmp/tidb_cdc_test/sink_retry + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/sink_retry/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/sink_retry/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/sink_retry/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/sink_retry/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/sink_retry/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/sink_retry/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/sink_retry/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/sink_retry/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/sink_retry/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/sink_retry/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:16:11 CST 2024] <<<<<< START pulsar cluster in normal mode in sink_retry case >>>>>>' [Sat May 4 21:16:11 CST 2024] <<<<<< START pulsar cluster in normal mode in sink_retry case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/sink_retry/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/sink_retry/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/sink_retry/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public public/default ++ date + echo '[Sat May 4 21:16:13 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:16:13 CST 2024] <<<<<< pulsar is ready >>>>>> table row_format.finish_mark not exists for 48-th check, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 public/default ++ date + echo '[Sat May 4 21:16:14 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:16:14 CST 2024] <<<<<< pulsar is ready >>>>>> Create changefeed successfully! ID: d47d3473-78a6-4c51-912f-d16c23525355 Info: {"upstream_id":7365132554912501842,"namespace":"default","id":"d47d3473-78a6-4c51-912f-d16c23525355","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T21:16:15.115507375+08:00","start_ts":449532019333922817,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532019333922817,"checkpoint_ts":449532019333922817,"checkpoint_time":"2024-05-04 21:16:04.964"} [Sat May 4 21:16:15 CST 2024] <<<<<< START Pulsar consumer in charset_gbk case >>>>>> public/default ++ date + echo '[Sat May 4 21:16:15 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:16:15 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 21:16:15 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.32623264.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Tenant not found Reason: Tenant not found [Sat May 4 21:16:15 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.33983400.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 table row_format.finish_mark not exists for 49-th check, retry later + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table row_format.finish_mark not exists for 50-th check, retry later public/default public/default + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > ++ date + echo '[Sat May 4 21:16:18 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:16:18 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.3338.out cli changefeed create --start-ts=449532019901202434 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-foreign-key-test-655?protocol=canal-json&enable-tidb-extension=true' + /usr/local/pulsar/bin/pulsar-admin namespaces list public Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 20.38 secs (183079622 bytes/sec) < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:16:18 GMT < Content-Length: 816 + (( i++ )) + (( i <= 50 )) < Content-Type: text/plain; charset=utf-8 < { [data not shown] ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 *** processors info ***: > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > *** etcd info ***: < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:16:18 GMT < Content-Length: 816 /tidb/cdc/default/__cdc_meta__/capture/741b6977-26e0-4106-8297-3a62a57e80dd < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/931eba93-7927-4d15-a056-b3f6a554ab57 {"id":"741b6977-26e0-4106-8297-3a62a57e80dd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828575} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf30037a 741b6977-26e0-4106-8297-3a62a57e80dd /tidb/cdc/default/default/upstream/7365132563769579560 {"id":"931eba93-7927-4d15-a056-b3f6a554ab57","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828575} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 {"id":7365132563769579560,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: /tidb/cdc/default/__cdc_meta__/owner/22318f43bf2944ec 931eba93-7927-4d15-a056-b3f6a554ab57 /tidb/cdc/default/default/upstream/7365132566024784931 *** processors info ***: {"id":7365132566024784931,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/741b6977-26e0-4106-8297-3a62a57e80dd /tidb/cdc/default/__cdc_meta__/capture/931eba93-7927-4d15-a056-b3f6a554ab57 {"id":"741b6977-26e0-4106-8297-3a62a57e80dd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828575} {"id":"931eba93-7927-4d15-a056-b3f6a554ab57","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828575} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf2944ec 931eba93-7927-4d15-a056-b3f6a554ab57 /tidb/cdc/default/default/upstream/7365132566024784931 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf30037a {"id":7365132566024784931,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: 741b6977-26e0-4106-8297-3a62a57e80dd /tidb/cdc/default/default/upstream/7365132563769579560 *** processors info ***: *** etcd info ***: {"id":7365132563769579560,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: /tidb/cdc/default/__cdc_meta__/capture/931eba93-7927-4d15-a056-b3f6a554ab57 *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/741b6977-26e0-4106-8297-3a62a57e80dd {"id":"931eba93-7927-4d15-a056-b3f6a554ab57","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828575} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf2944ec 931eba93-7927-4d15-a056-b3f6a554ab57 /tidb/cdc/default/default/upstream/7365132566024784931 {"id":"741b6977-26e0-4106-8297-3a62a57e80dd","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828575} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 {"id":7365132566024784931,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x /tidb/cdc/default/__cdc_meta__/owner/22318f43bf30037a 741b6977-26e0-4106-8297-3a62a57e80dd /tidb/cdc/default/default/upstream/7365132563769579560 {"id":7365132563769579560,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Pipeline] { [Pipeline] cache ++ date + echo '[Sat May 4 21:16:18 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:16:18 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.3354.out cli changefeed create --start-ts=449532019902251009 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4758?protocol=canal-json&enable-tidb-extension=true' [Sat May 4 21:16:18 CST 2024] <<<<<< START Pulsar consumer in gc_safepoint case >>>>>> 0 [Sat May 4 21:16:18 CST 2024] <<<<<< START Pulsar consumer in processor_err_chan case >>>>>> check_changefeed_state http://127.0.0.1:2379 eb03d211-532b-4e22-9bef-c9b3a13c9df9 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=eb03d211-532b-4e22-9bef-c9b3a13c9df9 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c eb03d211-532b-4e22-9bef-c9b3a13c9df9 -s + info='{ "upstream_id": 7365132563769579560, "namespace": "default", "id": "eb03d211-532b-4e22-9bef-c9b3a13c9df9", "state": "normal", "checkpoint_tso": 449532022815981572, "checkpoint_time": "2024-05-04 21:16:18.247", "error": null }' + echo '{ "upstream_id": 7365132563769579560, "namespace": "default", "id": "eb03d211-532b-4e22-9bef-c9b3a13c9df9", "state": "normal", "checkpoint_tso": 449532022815981572, "checkpoint_time": "2024-05-04 21:16:18.247", "error": null }' { "upstream_id": 7365132563769579560, "namespace": "default", "id": "eb03d211-532b-4e22-9bef-c9b3a13c9df9", "state": "normal", "checkpoint_tso": 449532022815981572, "checkpoint_time": "2024-05-04 21:16:18.247", "error": null } ++ echo '{' '"upstream_id":' 7365132563769579560, '"namespace":' '"default",' '"id":' '"eb03d211-532b-4e22-9bef-c9b3a13c9df9",' '"state":' '"normal",' '"checkpoint_tso":' 449532022815981572, '"checkpoint_time":' '"2024-05-04' '21:16:18.247",' '"error":' null '}' ++ jq -r .state Create changefeed successfully! ID: e7d89fac-3a82-4d3a-ae1f-6b347e3156b1 Info: {"upstream_id":7365132555989855257,"namespace":"default","id":"e7d89fac-3a82-4d3a-ae1f-6b347e3156b1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-foreign-key-test-655?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:16:19.051871027+08:00","start_ts":449532019901202434,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532019901202434,"checkpoint_ts":449532019901202434,"checkpoint_time":"2024-05-04 21:16:07.128"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365132563769579560, '"namespace":' '"default",' '"id":' '"eb03d211-532b-4e22-9bef-c9b3a13c9df9",' '"state":' '"normal",' '"checkpoint_tso":' 449532022815981572, '"checkpoint_time":' '"2024-05-04' '21:16:18.247",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully Create changefeed successfully! ID: 200b18cb-13d2-4ba7-8519-74b042e9540e Info: {"upstream_id":7365132547131950924,"namespace":"default","id":"200b18cb-13d2-4ba7-8519-74b042e9540e","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-4758?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:16:19.270715746+08:00","start_ts":449532019902251009,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532019902251009,"checkpoint_ts":449532019902251009,"checkpoint_time":"2024-05-04 21:16:07.132"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later table row_format.finish_mark not exists for 51-th check, retry later public/default ++ date + echo '[Sat May 4 21:16:20 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:16:20 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 21:16:20 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.33413343.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x [Sat May 4 21:16:20 CST 2024] <<<<<< START Pulsar consumer in foreign_key case >>>>>> + set +x [Sat May 4 21:16:20 CST 2024] <<<<<< START Pulsar consumer in sink_retry case >>>>>> table row_format.finish_mark not exists for 52-th check, retry later check diff failed 2-th time, retry later check diff failed 2-th time, retry later table foreign_key.finish_mark not exists for 1-th check, retry later table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:16:23 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bf54bcd-04d8-46b4-b737-0f4dd9982a5c {"id":"7bf54bcd-04d8-46b4-b737-0f4dd9982a5c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828580} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf37a4f7 7bf54bcd-04d8-46b4-b737-0f4dd9982a5c /tidb/cdc/default/default/upstream/7365132571311300587 {"id":7365132571311300587,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bf54bcd-04d8-46b4-b737-0f4dd9982a5c {"id":"7bf54bcd-04d8-46b4-b737-0f4dd9982a5c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828580} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf37a4f7 7bf54bcd-04d8-46b4-b737-0f4dd9982a5c /tidb/cdc/default/default/upstream/7365132571311300587 {"id":7365132571311300587,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7bf54bcd-04d8-46b4-b737-0f4dd9982a5c {"id":"7bf54bcd-04d8-46b4-b737-0f4dd9982a5c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828580} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bf37a4f7 7bf54bcd-04d8-46b4-b737-0f4dd9982a5c /tidb/cdc/default/default/upstream/7365132571311300587 {"id":7365132571311300587,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table row_format.finish_mark not exists for 53-th check, retry later check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365132566024784931 449532023844634629 449532022809165836 [Sat May 4 21:16:23 CST 2024] <<<<<< START Pulsar consumer in kv_client_stream_reconnect case >>>>>> check diff successfully wait process cdc.test exit for 1-th time... run task successfully check_changefeed_state http://127.0.0.1:2379 46bee176-3275-498c-9a52-b77dac4e4504 stopped null table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later wait process cdc.test exit for 2-th time... + endpoints=http://127.0.0.1:2379 + changefeed_id=46bee176-3275-498c-9a52-b77dac4e4504 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 46bee176-3275-498c-9a52-b77dac4e4504 -s + info='{ "upstream_id": 7365132566024784931, "namespace": "default", "id": "46bee176-3275-498c-9a52-b77dac4e4504", "state": "stopped", "checkpoint_tso": 449532024368660486, "checkpoint_time": "2024-05-04 21:16:24.170", "error": null }' + echo '{ "upstream_id": 7365132566024784931, "namespace": "default", "id": "46bee176-3275-498c-9a52-b77dac4e4504", "state": "stopped", "checkpoint_tso": 449532024368660486, "checkpoint_time": "2024-05-04 21:16:24.170", "error": null }' { "upstream_id": 7365132566024784931, "namespace": "default", "id": "46bee176-3275-498c-9a52-b77dac4e4504", "state": "stopped", "checkpoint_tso": 449532024368660486, "checkpoint_time": "2024-05-04 21:16:24.170", "error": null } ++ echo '{' '"upstream_id":' 7365132566024784931, '"namespace":' '"default",' '"id":' '"46bee176-3275-498c-9a52-b77dac4e4504",' '"state":' '"stopped",' '"checkpoint_tso":' 449532024368660486, '"checkpoint_time":' '"2024-05-04' '21:16:24.170",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365132566024784931, '"namespace":' '"default",' '"id":' '"46bee176-3275-498c-9a52-b77dac4e4504",' '"state":' '"stopped",' '"checkpoint_tso":' 449532024368660486, '"checkpoint_time":' '"2024-05-04' '21:16:24.170",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365132566024784931 table foreign_key.finish_mark not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 21:16:25 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> table row_format.finish_mark not exists for 54-th check, retry later table test.finish_mark not exists for 2-th check, retry later table foreign_key.finish_mark not exists for 3-th check, retry later table row_format.finish_mark not exists for 55-th check, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 46bee176-3275-498c-9a52-b77dac4e4504 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=46bee176-3275-498c-9a52-b77dac4e4504 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 46bee176-3275-498c-9a52-b77dac4e4504 -s table test.finish_mark not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 4-th check, retry later + info='{ "upstream_id": 7365132566024784931, "namespace": "default", "id": "46bee176-3275-498c-9a52-b77dac4e4504", "state": "normal", "checkpoint_tso": 449532024368660486, "checkpoint_time": "2024-05-04 21:16:24.170", "error": null }' + echo '{ "upstream_id": 7365132566024784931, "namespace": "default", "id": "46bee176-3275-498c-9a52-b77dac4e4504", "state": "normal", "checkpoint_tso": 449532024368660486, "checkpoint_time": "2024-05-04 21:16:24.170", "error": null }' { "upstream_id": 7365132566024784931, "namespace": "default", "id": "46bee176-3275-498c-9a52-b77dac4e4504", "state": "normal", "checkpoint_tso": 449532024368660486, "checkpoint_time": "2024-05-04 21:16:24.170", "error": null } ++ echo '{' '"upstream_id":' 7365132566024784931, '"namespace":' '"default",' '"id":' '"46bee176-3275-498c-9a52-b77dac4e4504",' '"state":' '"normal",' '"checkpoint_tso":' 449532024368660486, '"checkpoint_time":' '"2024-05-04' '21:16:24.170",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365132566024784931, '"namespace":' '"default",' '"id":' '"46bee176-3275-498c-9a52-b77dac4e4504",' '"state":' '"normal",' '"checkpoint_tso":' 449532024368660486, '"checkpoint_time":' '"2024-05-04' '21:16:24.170",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365132566024784931 449532024368660485 449532024368660486 table row_format.finish_mark not exists for 56-th check, retry later table test.finish_mark not exists for 4-th check, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 46bee176-3275-498c-9a52-b77dac4e4504 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=46bee176-3275-498c-9a52-b77dac4e4504 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 46bee176-3275-498c-9a52-b77dac4e4504 -s + info='{ "upstream_id": 7365132566024784931, "namespace": "default", "id": "46bee176-3275-498c-9a52-b77dac4e4504", "state": "stopped", "checkpoint_tso": 449532024368660486, "checkpoint_time": "2024-05-04 21:16:24.170", "error": null }' + echo '{ "upstream_id": 7365132566024784931, "namespace": "default", "id": "46bee176-3275-498c-9a52-b77dac4e4504", "state": "stopped", "checkpoint_tso": 449532024368660486, "checkpoint_time": "2024-05-04 21:16:24.170", "error": null }' { "upstream_id": 7365132566024784931, "namespace": "default", "id": "46bee176-3275-498c-9a52-b77dac4e4504", "state": "stopped", "checkpoint_tso": 449532024368660486, "checkpoint_time": "2024-05-04 21:16:24.170", "error": null } ++ echo '{' '"upstream_id":' 7365132566024784931, '"namespace":' '"default",' '"id":' '"46bee176-3275-498c-9a52-b77dac4e4504",' '"state":' '"stopped",' '"checkpoint_tso":' 449532024368660486, '"checkpoint_time":' '"2024-05-04' '21:16:24.170",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365132566024784931, '"namespace":' '"default",' '"id":' '"46bee176-3275-498c-9a52-b77dac4e4504",' '"state":' '"stopped",' '"checkpoint_tso":' 449532024368660486, '"checkpoint_time":' '"2024-05-04' '21:16:24.170",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 ef4f3789-9ed2-4191-8453-382f3c5817d7 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=ef4f3789-9ed2-4191-8453-382f3c5817d7 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c ef4f3789-9ed2-4191-8453-382f3c5817d7 -s + info='{ "upstream_id": 7365132566024784931, "namespace": "default", "id": "ef4f3789-9ed2-4191-8453-382f3c5817d7", "state": "normal", "checkpoint_tso": 449532026099073027, "checkpoint_time": "2024-05-04 21:16:30.771", "error": null }' + echo '{ "upstream_id": 7365132566024784931, "namespace": "default", "id": "ef4f3789-9ed2-4191-8453-382f3c5817d7", "state": "normal", "checkpoint_tso": 449532026099073027, "checkpoint_time": "2024-05-04 21:16:30.771", "error": null }' { "upstream_id": 7365132566024784931, "namespace": "default", "id": "ef4f3789-9ed2-4191-8453-382f3c5817d7", "state": "normal", "checkpoint_tso": 449532026099073027, "checkpoint_time": "2024-05-04 21:16:30.771", "error": null } ++ echo '{' '"upstream_id":' 7365132566024784931, '"namespace":' '"default",' '"id":' '"ef4f3789-9ed2-4191-8453-382f3c5817d7",' '"state":' '"normal",' '"checkpoint_tso":' 449532026099073027, '"checkpoint_time":' '"2024-05-04' '21:16:30.771",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365132566024784931, '"namespace":' '"default",' '"id":' '"ef4f3789-9ed2-4191-8453-382f3c5817d7",' '"state":' '"normal",' '"checkpoint_tso":' 449532026099073027, '"checkpoint_time":' '"2024-05-04' '21:16:30.771",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365132566024784931 table foreign_key.finish_mark not exists for 5-th check, retry later table row_format.finish_mark not exists for 57-th check, retry later table test.finish_mark not exists for 5-th check, retry later table foreign_key.finish_mark not exists for 6-th check, retry later table row_format.finish_mark not exists for 58-th check, retry later run task successfully Changefeed remove successfully. ID: 46bee176-3275-498c-9a52-b77dac4e4504 CheckpointTs: 449532024368660486 SinkURI: pulsar://127.0.0.1:6650/ticdc-gc-safepoint-31989?protocol=canal-json&enable-tidb-extension=true check_safepoint_forward http://127.0.0.1:2379 7365132566024784931 449532026990362631 449532024368660486 449532026099073027 table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... table foreign_key.finish_mark not exists for 7-th check, retry later table row_format.finish_mark not exists for 59-th check, retry later wait process cdc.test exit for 2-th time... run task successfully cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 21:16:35 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> Changefeed remove successfully. ID: ef4f3789-9ed2-4191-8453-382f3c5817d7 CheckpointTs: 449532027252506631 SinkURI: pulsar://127.0.0.1:6650/ticdc-gc-safepoint-31989?protocol=canal-json&enable-tidb-extension=true check_safepoint_cleared http://127.0.0.1:2379 7365132566024784931 run task successfully wait process cdc.test exit for 1-th time... table foreign_key.finish_mark not exists for 8-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 16.57 secs (225111906 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 21:16:37 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> table row_format.finish_mark not exists for 60-th check, retry later table foreign_key.finish_mark not exists for 9-th check, retry later table row_format.finish_mark not exists at last check =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... table foreign_key.finish_mark not exists for 10-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table foreign_key.finish_mark not exists for 11-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table foreign_key.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 6.44 secs (579480997 bytes/sec) [Pipeline] { [Pipeline] cache wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 21:16:46 CST 2024] <<<<<< run test case foreign_key success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f003dd0000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q, pid:4875, start at 2024-05-04 21:16:51.456053544 +0800 CST m=+5.608108862 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:51.466 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:51.444 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:51.444 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f003dd0000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q, pid:4875, start at 2024-05-04 21:16:51.456053544 +0800 CST m=+5.608108862 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:51.466 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:51.444 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:51.444 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f003a880014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q, pid:4955, start at 2024-05-04 21:16:51.264042354 +0800 CST m=+5.360866806 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:51.273 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:51.234 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:51.234 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "dotransactions"="false" "threadcount"="2" "mysql.host"="127.0.0.1" "scanproportion"="0" "mysql.db"="sink_retry" "readproportion"="0" "mysql.port"="4000" "updateproportion"="0" "insertproportion"="0" "recordcount"="10" "requestdistribution"="uniform" "mysql.user"="root" "workload"="core" "operationcount"="0" "readallfields"="true" ********************************************** Run finished, takes 4.926474ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2886.2, Avg(us): 876, Min(us): 513, Max(us): 2099, 95th(us): 3000, 99th(us): 3000 Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Sat May 4 21:16:54 CST 2024] <<<<<< START pulsar cluster in changefeed_reconstruct case >>>>>>' [Sat May 4 21:16:54 CST 2024] <<<<<< START pulsar cluster in changefeed_reconstruct case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_reconstruct + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_reconstruct + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_reconstruct/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_reconstruct/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_reconstruct/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:16:54 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_reconstruct case >>>>>>' [Sat May 4 21:16:54 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_reconstruct case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_reconstruct/pulsar-bookie + nc -z localhost 6650 + i=1 + '[' 1 -gt 20 ']' + sleep 2 check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 21:17:00 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f00be6c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm, pid:4637, start at 2024-05-04 21:16:59.676168507 +0800 CST m=+5.162661503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:59.683 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:59.675 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:59.675 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f00db8c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8, pid:5358, start at 2024-05-04 21:17:01.558394503 +0800 CST m=+5.250600554 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:01.565 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:01.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:01.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f00be6c0003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm, pid:4637, start at 2024-05-04 21:16:59.676168507 +0800 CST m=+5.162661503 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:59.683 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:59.675 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:59.675 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f00bf200014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-v4bw-wtjdm, pid:4717, start at 2024-05-04 21:16:59.750508821 +0800 CST m=+5.185709842 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:18:59.757 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:16:59.720 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:06:59.720 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Tenant not found Reason: Tenant not found + i=1 + '[' 1 -gt 20 ']' + sleep 2 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 16.86 secs (221270983 bytes/sec) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f00db8c0011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8, pid:5358, start at 2024-05-04 21:17:01.558394503 +0800 CST m=+5.250600554 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:01.565 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:01.539 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:01.539 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f00dc2c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-11rq-rbfc8, pid:5440, start at 2024-05-04 21:17:01.618965433 +0800 CST m=+5.259465030 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:01.624 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:01.629 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:01.629 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] { [Sat May 4 21:17:04 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.60476049.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] cache + /usr/local/pulsar/bin/pulsar-admin namespaces list public Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ++ date + echo '[Sat May 4 21:17:06 CST 2024] <<<<<< START pulsar cluster in changefeed_pause_resume case >>>>>>' [Sat May 4 21:17:06 CST 2024] <<<<<< START pulsar cluster in changefeed_pause_resume case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_pause_resume + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_pause_resume + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_pause_resume/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_pause_resume/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_pause_resume/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_pause_resume/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:17:06 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_pause_resume case >>>>>>' [Sat May 4 21:17:06 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_pause_resume case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_pause_resume/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:17:07 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/408ab814-cb4e-46ab-b7cb-c608d1adce37 {"id":"408ab814-cb4e-46ab-b7cb-c608d1adce37","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828625} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0059ac9 408ab814-cb4e-46ab-b7cb-c608d1adce37 /tidb/cdc/default/default/upstream/7365132803632675354 {"id":7365132803632675354,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/408ab814-cb4e-46ab-b7cb-c608d1adce37 {"id":"408ab814-cb4e-46ab-b7cb-c608d1adce37","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828625} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0059ac9 408ab814-cb4e-46ab-b7cb-c608d1adce37 /tidb/cdc/default/default/upstream/7365132803632675354 {"id":7365132803632675354,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/408ab814-cb4e-46ab-b7cb-c608d1adce37 {"id":"408ab814-cb4e-46ab-b7cb-c608d1adce37","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828625} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0059ac9 408ab814-cb4e-46ab-b7cb-c608d1adce37 /tidb/cdc/default/default/upstream/7365132803632675354 {"id":7365132803632675354,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:17:07 CST 2024] <<<<<< START pulsar cluster in ddl_manager case >>>>>>' [Sat May 4 21:17:07 CST 2024] <<<<<< START pulsar cluster in ddl_manager case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_manager + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_manager + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_manager/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_manager/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_manager/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_manager/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_manager/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_manager/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_manager/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_manager/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_manager/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_manager/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:17:07 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_manager case >>>>>>' [Sat May 4 21:17:07 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_manager case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_manager/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_manager/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_manager/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) public/default ++ date + echo '[Sat May 4 21:17:08 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:17:08 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 21:17:08 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.67816783.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + i=3 + '[' 3 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:17:11 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee0dfb3c-f311-4cea-a732-245beb5510c1 {"id":"ee0dfb3c-f311-4cea-a732-245beb5510c1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828628} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bfe98a03 ee0dfb3c-f311-4cea-a732-245beb5510c1 /tidb/cdc/default/default/upstream/7365132765103576632 {"id":7365132765103576632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee0dfb3c-f311-4cea-a732-245beb5510c1 {"id":"ee0dfb3c-f311-4cea-a732-245beb5510c1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828628} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bfe98a03 ee0dfb3c-f311-4cea-a732-245beb5510c1 /tidb/cdc/default/default/upstream/7365132765103576632 {"id":7365132765103576632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ee0dfb3c-f311-4cea-a732-245beb5510c1 {"id":"ee0dfb3c-f311-4cea-a732-245beb5510c1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828628} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bfe98a03 ee0dfb3c-f311-4cea-a732-245beb5510c1 /tidb/cdc/default/default/upstream/7365132765103576632 {"id":7365132765103576632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Sat May 4 21:17:12 CST 2024] <<<<<< START Pulsar consumer in changefeed_reconstruct case >>>>>> ***************** properties ***************** "recordcount"="50" "threadcount"="4" "dotransactions"="false" "requestdistribution"="uniform" "mysql.port"="4000" "workload"="core" "mysql.host"="127.0.0.1" "readproportion"="0" "readallfields"="true" "mysql.user"="root" "mysql.db"="changefeed_reconstruct" "operationcount"="0" "insertproportion"="0" "scanproportion"="0" "updateproportion"="0" ********************************************** Run finished, takes 20.099837ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3112.5, Avg(us): 1602, Min(us): 1089, Max(us): 4590, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0179a00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p, pid:4611, start at 2024-05-04 21:17:11.659271923 +0800 CST m=+5.511176818 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:11.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:11.656 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:11.656 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public public/default ++ date + echo '[Sat May 4 21:17:14 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:17:14 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.6515.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json&enable-tidb-extension=true' -c=ddl-manager VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0179a00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p, pid:4611, start at 2024-05-04 21:17:11.659271923 +0800 CST m=+5.511176818 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:11.667 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:11.656 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:11.656 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0178000013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-30rn-8nk6p, pid:4698, start at 2024-05-04 21:17:11.588818243 +0800 CST m=+5.386255772 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:11.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:11.653 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:11.653 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Tenant not found Reason: Tenant not found table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365132803632675354,"namespace":"default","id":"ddl-manager","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:17:15.049318066+08:00","start_ts":449532037542969349,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532037542969349,"checkpoint_ts":449532037542969349,"checkpoint_time":"2024-05-04 21:17:14.426"} PASS + i=1 + '[' 1 -gt 20 ']' + sleep 2 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: pulsar... <<================= coverage: 2.4% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.6076.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + set +x [Sat May 4 21:17:16 CST 2024] <<<<<< START Pulsar consumer in ddl_manager case >>>>>> check diff successfully + /usr/local/pulsar/bin/pulsar-admin namespaces list public + set +x + tso='449532038166872065 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532038166872065 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 21:17:18 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.61266128.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 public/default wait process 6786 exit for 1-th time... ++ date + echo '[Sat May 4 21:17:19 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:17:19 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 21:17:19 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.72947296.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (6786) - No such process wait process 6786 exit for 2-th time... process 6786 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sat May 4 21:17:19 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.71107112.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process 6052 exit for 1-th time... wait process 6052 exit for 2-th time... wait process 6052 exit for 3-th time... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 14.23 secs (262155265 bytes/sec) [Pipeline] { [Pipeline] cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:17:21 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e74bf954-65f4-43ca-b8c4-e36b6aa75264 {"id":"e74bf954-65f4-43ca-b8c4-e36b6aa75264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828638} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c03112f0 e74bf954-65f4-43ca-b8c4-e36b6aa75264 /tidb/cdc/default/default/upstream/7365132854748500809 {"id":7365132854748500809,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e74bf954-65f4-43ca-b8c4-e36b6aa75264 {"id":"e74bf954-65f4-43ca-b8c4-e36b6aa75264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828638} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c03112f0 e74bf954-65f4-43ca-b8c4-e36b6aa75264 /tidb/cdc/default/default/upstream/7365132854748500809 {"id":7365132854748500809,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e74bf954-65f4-43ca-b8c4-e36b6aa75264 {"id":"e74bf954-65f4-43ca-b8c4-e36b6aa75264","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828638} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c03112f0 e74bf954-65f4-43ca-b8c4-e36b6aa75264 /tidb/cdc/default/default/upstream/7365132854748500809 {"id":7365132854748500809,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:17:21 CST 2024] <<<<<< START pulsar cluster in ddl_puller_lag case >>>>>>' [Sat May 4 21:17:21 CST 2024] <<<<<< START pulsar cluster in ddl_puller_lag case >>>>>> + workdir=/tmp/tidb_cdc_test/ddl_puller_lag + cluster_type=normal + cd /tmp/tidb_cdc_test/ddl_puller_lag + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/ddl_puller_lag/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/ddl_puller_lag/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/ddl_puller_lag/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/ddl_puller_lag/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/ddl_puller_lag/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:17:21 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_puller_lag case >>>>>>' [Sat May 4 21:17:21 CST 2024] <<<<<< START pulsar cluster in normal mode in ddl_puller_lag case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/ddl_puller_lag/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/ddl_puller_lag/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/ddl_puller_lag/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 wait process 6052 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (6052) - No such process wait process 6052 exit for 5-th time... process 6052 already exit [Sat May 4 21:17:22 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.66536655.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:17:22 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fc88124-e2d9-441d-8881-08904df7e168 {"id":"1fc88124-e2d9-441d-8881-08904df7e168","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c00c7e08 1fc88124-e2d9-441d-8881-08904df7e168 /tidb/cdc/default/default/upstream/7365132810863722772 {"id":7365132810863722772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fc88124-e2d9-441d-8881-08904df7e168 {"id":"1fc88124-e2d9-441d-8881-08904df7e168","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c00c7e08 1fc88124-e2d9-441d-8881-08904df7e168 /tidb/cdc/default/default/upstream/7365132810863722772 {"id":7365132810863722772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1fc88124-e2d9-441d-8881-08904df7e168 {"id":"1fc88124-e2d9-441d-8881-08904df7e168","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828639} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c00c7e08 1fc88124-e2d9-441d-8881-08904df7e168 /tidb/cdc/default/default/upstream/7365132810863722772 {"id":7365132810863722772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:17:22 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/01da9034-90fa-4171-9845-fe49279ff196 {UpstreamID:7365132765103576632 Namespace:default ID:01da9034-90fa-4171-9845-fe49279ff196 SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-831?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:17:12.293618251 +0800 CST StartTs:449532036800577540 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014b8ab0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532036970971142} {CheckpointTs:449532039500660741 MinTableBarrierTs:449532039539720195 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449532039500660741, checkpointTs: 449532039500660741, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc882bde-dc2d-4c5f-b5d3-11eec11ba7af {"id":"bc882bde-dc2d-4c5f-b5d3-11eec11ba7af","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828640} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bfe98aaf bc882bde-dc2d-4c5f-b5d3-11eec11ba7af /tidb/cdc/default/default/changefeed/info/01da9034-90fa-4171-9845-fe49279ff196 {"upstream-id":7365132765103576632,"namespace":"default","changefeed-id":"01da9034-90fa-4171-9845-fe49279ff196","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-831?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:17:12.293618251+08:00","start-ts":449532036800577540,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532036970971142} /tidb/cdc/default/default/changefeed/status/01da9034-90fa-4171-9845-fe49279ff196 {"checkpoint-ts":449532039500660741,"min-table-barrier-ts":449532039539720195,"admin-job-type":0} /tidb/cdc/default/default/task/position/bc882bde-dc2d-4c5f-b5d3-11eec11ba7af/01da9034-90fa-4171-9845-fe49279ff196 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365132765103576632 {"id":7365132765103576632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/01da9034-90fa-4171-9845-fe49279ff196 {UpstreamID:7365132765103576632 Namespace:default ID:01da9034-90fa-4171-9845-fe49279ff196 SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-831?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:17:12.293618251 +0800 CST StartTs:449532036800577540 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014b8ab0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532036970971142} {CheckpointTs:449532039500660741 MinTableBarrierTs:449532039539720195 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449532039500660741, checkpointTs: 449532039500660741, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc882bde-dc2d-4c5f-b5d3-11eec11ba7af {"id":"bc882bde-dc2d-4c5f-b5d3-11eec11ba7af","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828640} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bfe98aaf bc882bde-dc2d-4c5f-b5d3-11eec11ba7af /tidb/cdc/default/default/changefeed/info/01da9034-90fa-4171-9845-fe49279ff196 {"upstream-id":7365132765103576632,"namespace":"default","changefeed-id":"01da9034-90fa-4171-9845-fe49279ff196","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-831?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:17:12.293618251+08:00","start-ts":449532036800577540,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532036970971142} /tidb/cdc/default/default/changefeed/status/01da9034-90fa-4171-9845-fe49279ff196 {"checkpoint-ts":449532039500660741,"min-table-barrier-ts":449532039539720195,"admin-job-type":0} /tidb/cdc/default/default/task/position/bc882bde-dc2d-4c5f-b5d3-11eec11ba7af/01da9034-90fa-4171-9845-fe49279ff196 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365132765103576632 {"id":7365132765103576632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/01da9034-90fa-4171-9845-fe49279ff196 {UpstreamID:7365132765103576632 Namespace:default ID:01da9034-90fa-4171-9845-fe49279ff196 SinkURI:pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-831?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:17:12.293618251 +0800 CST StartTs:449532036800577540 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014b8ab0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532036970971142} {CheckpointTs:449532039500660741 MinTableBarrierTs:449532039539720195 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449532039500660741, checkpointTs: 449532039500660741, state: Replicating *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc882bde-dc2d-4c5f-b5d3-11eec11ba7af {"id":"bc882bde-dc2d-4c5f-b5d3-11eec11ba7af","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828640} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43bfe98aaf bc882bde-dc2d-4c5f-b5d3-11eec11ba7af /tidb/cdc/default/default/changefeed/info/01da9034-90fa-4171-9845-fe49279ff196 {"upstream-id":7365132765103576632,"namespace":"default","changefeed-id":"01da9034-90fa-4171-9845-fe49279ff196","sink-uri":"pulsar://127.0.0.1:6650/ticdc-changefeed-reconstruct-831?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:17:12.293618251+08:00","start-ts":449532036800577540,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532036970971142} /tidb/cdc/default/default/changefeed/status/01da9034-90fa-4171-9845-fe49279ff196 {"checkpoint-ts":449532039500660741,"min-table-barrier-ts":449532039539720195,"admin-job-type":0} /tidb/cdc/default/default/task/position/bc882bde-dc2d-4c5f-b5d3-11eec11ba7af/01da9034-90fa-4171-9845-fe49279ff196 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365132765103576632 + grep -q 'etcd info' {"id":7365132765103576632,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id [Sat May 4 21:17:23 CST 2024] <<<<<< START Pulsar consumer in changefeed_pause_resume case >>>>>> + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table changefeed_pause_resume.t1 not exists for 1-th check, retry later "id": "bc882bde-dc2d-4c5f-b5d3-11eec11ba7af", "cluster-id": "default" run task successfully table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 21:17:23 CST 2024] <<<<<< run test case sink_retry success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 not exists for 1-th check, retry later capture_id: bc882bde-dc2d-4c5f-b5d3-11eec11ba7af check_processor_table_count http://127.0.0.1:2379 01da9034-90fa-4171-9845-fe49279ff196 bc882bde-dc2d-4c5f-b5d3-11eec11ba7af 1 run task successfully check_processor_table_count http://127.0.0.1:2379 01da9034-90fa-4171-9845-fe49279ff196 bc882bde-dc2d-4c5f-b5d3-11eec11ba7af 0 start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table count 1 does equal to expected count 0 run task failed 1-th time, retry later public/default Verifying downstream PD is started... ++ date + echo '[Sat May 4 21:17:27 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:17:27 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.6601.out cli changefeed create --start-ts=449532038166872065 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-11911?protocol=canal-json&enable-tidb-extension=true' < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:17:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365132803632675354 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:17:15.049318066 +0800 CST StartTs:449532037542969349 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025ca6c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532037700255747} {CheckpointTs:449532038093209620 MinTableBarrierTs:449532038093209620 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b85a016e-efbe-4b25-aa2c-df606f825ab7 {"id":"b85a016e-efbe-4b25-aa2c-df606f825ab7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828642} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0059d76 b85a016e-efbe-4b25-aa2c-df606f825ab7 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365132803632675354,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:17:15.049318066+08:00","start-ts":449532037542969349,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532037700255747} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449532038093209620,"min-table-barrier-ts":449532038093209620,"admin-job-type":0} /tidb/cdc/default/default/task/position/b85a016e-efbe-4b25-aa2c-df606f825ab7/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365132803632675354 {"id":7365132803632675354,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365132803632675354 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:17:15.049318066 +0800 CST StartTs:449532037542969349 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025ca6c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532037700255747} {CheckpointTs:449532038093209620 MinTableBarrierTs:449532038093209620 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b85a016e-efbe-4b25-aa2c-df606f825ab7 {"id":"b85a016e-efbe-4b25-aa2c-df606f825ab7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828642} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0059d76 b85a016e-efbe-4b25-aa2c-df606f825ab7 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365132803632675354,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:17:15.049318066+08:00","start-ts":449532037542969349,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532037700255747} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449532038093209620,"min-table-barrier-ts":449532038093209620,"admin-job-type":0} /tidb/cdc/default/default/task/position/b85a016e-efbe-4b25-aa2c-df606f825ab7/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365132803632675354 {"id":7365132803632675354,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365132803632675354 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:17:15.049318066 +0800 CST StartTs:449532037542969349 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025ca6c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532037700255747} {CheckpointTs:449532038093209620 MinTableBarrierTs:449532038093209620 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b85a016e-efbe-4b25-aa2c-df606f825ab7 {"id":"b85a016e-efbe-4b25-aa2c-df606f825ab7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828642} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0059d76 b85a016e-efbe-4b25-aa2c-df606f825ab7 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365132803632675354,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:17:15.049318066+08:00","start-ts":449532037542969349,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532037700255747} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449532038093209620,"min-table-barrier-ts":449532038093209620,"admin-job-type":0} /tidb/cdc/default/default/task/position/b85a016e-efbe-4b25-aa2c-df606f825ab7/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365132803632675354 + grep -q 'etcd info' {"id":7365132803632675354,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Sat May 4 21:17:27 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.67196721.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 not exists for 1-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_processor_table_count http://127.0.0.1:2379 01da9034-90fa-4171-9845-fe49279ff196 bc882bde-dc2d-4c5f-b5d3-11eec11ba7af 0 run task successfully ***************** properties ***************** "readproportion"="0" "scanproportion"="0" "threadcount"="4" "operationcount"="0" "dotransactions"="false" "insertproportion"="0" "requestdistribution"="uniform" "workload"="core" "updateproportion"="0" "mysql.db"="changefeed_reconstruct" "mysql.port"="4000" "recordcount"="50" "readallfields"="true" "mysql.host"="127.0.0.1" "mysql.user"="root" ********************************************** Run finished, takes 15.818214ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3951.0, Avg(us): 1236, Min(us): 873, Max(us): 3748, 95th(us): 4000, 99th(us): 4000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later Create changefeed successfully! ID: e71c51bd-485b-416c-8d26-d3a677d72614 Info: {"upstream_id":7365132854748500809,"namespace":"default","id":"e71c51bd-485b-416c-8d26-d3a677d72614","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-11911?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:17:28.861371542+08:00","start_ts":449532038166872065,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532038166872065,"checkpoint_ts":449532038166872065,"checkpoint_time":"2024-05-04 21:17:16.806"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table changefeed_pause_resume.t3 exists + set +x [Sat May 4 21:17:30 CST 2024] <<<<<< START Pulsar consumer in ddl_puller_lag case >>>>>> check diff failed 1-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_reconstruct.usertable exists check diff failed 1-th time, retry later < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:17:31 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365132803632675354 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:17:15.049318066 +0800 CST StartTs:449532037542969349 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025ca6c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532037700255747} {CheckpointTs:449532038093209620 MinTableBarrierTs:449532038093209620 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b85a016e-efbe-4b25-aa2c-df606f825ab7 {"id":"b85a016e-efbe-4b25-aa2c-df606f825ab7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828642} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0059d76 b85a016e-efbe-4b25-aa2c-df606f825ab7 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365132803632675354,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:17:15.049318066+08:00","start-ts":449532037542969349,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532037700255747} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449532038119424021,"min-table-barrier-ts":449532038119424021,"admin-job-type":0} /tidb/cdc/default/default/task/position/b85a016e-efbe-4b25-aa2c-df606f825ab7/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365132803632675354 {"id":7365132803632675354,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365132803632675354 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:17:15.049318066 +0800 CST StartTs:449532037542969349 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025ca6c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532037700255747} {CheckpointTs:449532038093209620 MinTableBarrierTs:449532038093209620 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b85a016e-efbe-4b25-aa2c-df606f825ab7 {"id":"b85a016e-efbe-4b25-aa2c-df606f825ab7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828642} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0059d76 b85a016e-efbe-4b25-aa2c-df606f825ab7 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365132803632675354,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:17:15.049318066+08:00","start-ts":449532037542969349,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532037700255747} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449532038119424021,"min-table-barrier-ts":449532038119424021,"admin-job-type":0} /tidb/cdc/default/default/task/position/b85a016e-efbe-4b25-aa2c-df606f825ab7/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365132803632675354 + grep -q 'failed to get info:' {"id":7365132803632675354,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365132803632675354 Namespace:default ID:ddl-manager SinkURI:pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:17:15.049318066 +0800 CST StartTs:449532037542969349 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0025ca6c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532037700255747} {CheckpointTs:449532038093209620 MinTableBarrierTs:449532038093209620 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b85a016e-efbe-4b25-aa2c-df606f825ab7 {"id":"b85a016e-efbe-4b25-aa2c-df606f825ab7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828642} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0059d76 b85a016e-efbe-4b25-aa2c-df606f825ab7 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365132803632675354,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"pulsar://127.0.0.1:6650/ticdc-ddl-mamager-test-23275?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:17:15.049318066+08:00","start-ts":449532037542969349,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532037700255747} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449532038119424021,"min-table-barrier-ts":449532038119424021,"admin-job-type":0} /tidb/cdc/default/default/task/position/b85a016e-efbe-4b25-aa2c-df606f825ab7/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365132803632675354 {"id":7365132803632675354,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully check diff failed 2-th time, retry later wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 2-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 21:17:35 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 4-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 14.19 secs (262943312 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0300dc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p, pid:5030, start at 2024-05-04 21:17:36.696851711 +0800 CST m=+5.456207626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:36.703 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:36.695 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:36.695 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0300dc0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p, pid:5030, start at 2024-05-04 21:17:36.696851711 +0800 CST m=+5.456207626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:36.703 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:36.695 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:36.695 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0300000008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-mbt1-5jf4p, pid:5112, start at 2024-05-04 21:17:36.648868019 +0800 CST m=+5.342023356 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:36.657 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:36.640 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:36.640 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 5-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.6497.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 6-th check, retry later + set +x + tso='449532044494503937 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532044494503937 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 21:17:42 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.65456547.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:17:45 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdf76362-07aa-4953-91f6-1991b7b5d757 {"id":"cdf76362-07aa-4953-91f6-1991b7b5d757","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828663} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c094d5fa cdf76362-07aa-4953-91f6-1991b7b5d757 /tidb/cdc/default/default/upstream/7365132953080930117 {"id":7365132953080930117,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdf76362-07aa-4953-91f6-1991b7b5d757 {"id":"cdf76362-07aa-4953-91f6-1991b7b5d757","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828663} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c094d5fa cdf76362-07aa-4953-91f6-1991b7b5d757 /tidb/cdc/default/default/upstream/7365132953080930117 {"id":7365132953080930117,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cdf76362-07aa-4953-91f6-1991b7b5d757 {"id":"cdf76362-07aa-4953-91f6-1991b7b5d757","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828663} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c094d5fa cdf76362-07aa-4953-91f6-1991b7b5d757 /tidb/cdc/default/default/upstream/7365132953080930117 {"id":7365132953080930117,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:17:45 CST 2024] <<<<<< START pulsar cluster in split_region case >>>>>>' [Sat May 4 21:17:45 CST 2024] <<<<<< START pulsar cluster in split_region case >>>>>> + workdir=/tmp/tidb_cdc_test/split_region + cluster_type=oauth + cd /tmp/tidb_cdc_test/split_region + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/split_region/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/split_region/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/split_region/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/split_region/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/split_region/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/split_region/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/split_region/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/split_region/pulsar_standalone.conf + pulsar_port=6650 + '[' oauth == mtls ']' + '[' oauth == oauth ']' + echo 'Waiting for oauth2 server to be ready...' Waiting for oauth2 server to be ready... + i=0 + nc -z localhost 9096 + oauth2-server + gen_oauth_config + echo ' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/split_region/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ date + echo '[Sat May 4 21:17:45 CST 2024] <<<<<< START pulsar cluster in oauth mode in split_region case >>>>>>' [Sat May 4 21:17:45 CST 2024] <<<<<< START pulsar cluster in oauth mode in split_region case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/split_region/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/split_region/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/split_region/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 9-th check, retry later check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f03a9100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg, pid:4746, start at 2024-05-04 21:17:47.484579464 +0800 CST m=+5.185718626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:47.491 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:47.460 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:47.460 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table ddl_manager.finish_mark not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f03a9100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg, pid:4746, start at 2024-05-04 21:17:47.484579464 +0800 CST m=+5.185718626 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:47.491 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:47.460 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:47.460 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f03aa84000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-skmf-p0hvg, pid:4818, start at 2024-05-04 21:17:47.56438189 +0800 CST m=+5.211680468 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:19:47.571 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:17:47.553 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:07:47.553 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 11-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6137.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... public/default ++ date + echo '[Sat May 4 21:17:53 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:17:53 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.7029.out cli changefeed create --start-ts=449532044494503937 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-split-region-test-28990?protocol=canal-json&enable-tidb-extension=true' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 12-th check, retry later + set +x + tso='449532047657271297 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532047657271297 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x check diff successfully ***************** properties ***************** "readallfields"="true" "requestdistribution"="uniform" "mysql.user"="root" "mysql.host"="127.0.0.1" "dotransactions"="false" "operationcount"="0" "workload"="core" "readproportion"="0" "mysql.db"="changefeed_error" "threadcount"="4" "scanproportion"="0" "insertproportion"="0" "recordcount"="20" "mysql.port"="4000" "updateproportion"="0" ********************************************** Run finished, takes 11.771828ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2809.1, Avg(us): 2238, Min(us): 1300, Max(us): 4636, 95th(us): 5000, 99th(us): 5000 [Sat May 4 21:17:54 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.61986200.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 9692ab4c-532c-4316-80e7-983e62082c10 Info: {"upstream_id":7365132953080930117,"namespace":"default","id":"9692ab4c-532c-4316-80e7-983e62082c10","sink_uri":"pulsar://127.0.0.1:6650/ticdc-split-region-test-28990?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:17:54.900686012+08:00","start_ts":449532044494503937,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"oauth2":{"oauth2-issuer-url":"http://localhost:9096","oauth2-audience":"cdc-api-uri","oauth2-private-key":"/tmp/tidb_cdc_test/split_region/credential.json","oauth2-client-id":"1234"}},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532044494503937,"checkpoint_ts":449532044494503937,"checkpoint_time":"2024-05-04 21:17:40.944"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 13-th check, retry later Verifying downstream PD is started... + set +x [Sat May 4 21:17:56 CST 2024] <<<<<< START Pulsar consumer in split_region case >>>>>> table split_region.test1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:17:57 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 {"id":"1a83b53b-2650-4200-96cc-d04a9a5fcb08","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828675} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0c040ed 1a83b53b-2650-4200-96cc-d04a9a5fcb08 /tidb/cdc/default/default/upstream/7365133008823964645 {"id":7365133008823964645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 {"id":"1a83b53b-2650-4200-96cc-d04a9a5fcb08","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828675} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0c040ed 1a83b53b-2650-4200-96cc-d04a9a5fcb08 /tidb/cdc/default/default/upstream/7365133008823964645 {"id":7365133008823964645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 {"id":"1a83b53b-2650-4200-96cc-d04a9a5fcb08","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828675} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0c040ed 1a83b53b-2650-4200-96cc-d04a9a5fcb08 /tidb/cdc/default/default/upstream/7365133008823964645 {"id":7365133008823964645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:17:57 CST 2024] <<<<<< START pulsar cluster in changefeed_error case >>>>>>' [Sat May 4 21:17:57 CST 2024] <<<<<< START pulsar cluster in changefeed_error case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_error + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_error + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_error/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_error/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_error/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_error/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_error/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_error/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_error/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_error/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_error/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_error/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:17:57 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_error case >>>>>>' [Sat May 4 21:17:57 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_error case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_error/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_error/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_error/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 14-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 17.90 secs (208379987 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table split_region.test1 exists table split_region.test2 not exists for 1-th check, retry later check diff failed 2-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 table ddl_manager.finish_mark not exists for 15-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table split_region.test2 exists check diff failed 3-th time, retry later check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public table ddl_manager.finish_mark not exists for 16-th check, retry later check diff successfully public/default ++ date + echo '[Sat May 4 21:18:04 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:18:04 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6672.out cli changefeed create --start-ts=449532047657271297 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-15128?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error table ddl_manager.finish_mark not exists for 17-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365133008823964645,"namespace":"default","id":"changefeed-error","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-15128?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:18:05.264290639+08:00","start_ts":449532047657271297,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532047657271297,"checkpoint_ts":449532047657271297,"checkpoint_time":"2024-05-04 21:17:53.009"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 18-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f04c0780018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q, pid:8632, start at 2024-05-04 21:18:05.392016986 +0800 CST m=+5.265250195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:05.399 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:05.392 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:05.392 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x [Sat May 4 21:18:06 CST 2024] <<<<<< START Pulsar consumer in changefeed_error case >>>>>> check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365133008823964645, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449532047657271297, "checkpoint_time": "2024-05-04 21:17:53.009", "error": { "time": "2024-05-04T21:18:05.373668238+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449532047657270997 is earlier than or equal to GC safepoint at 449532047657271297" } }' + echo '{ "upstream_id": 7365133008823964645, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449532047657271297, "checkpoint_time": "2024-05-04 21:17:53.009", "error": { "time": "2024-05-04T21:18:05.373668238+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449532047657270997 is earlier than or equal to GC safepoint at 449532047657271297" } }' { "upstream_id": 7365133008823964645, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449532047657271297, "checkpoint_time": "2024-05-04 21:17:53.009", "error": { "time": "2024-05-04T21:18:05.373668238+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449532047657270997 is earlier than or equal to GC safepoint at 449532047657271297" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365133008823964645, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449532047657271297, '"checkpoint_time":' '"2024-05-04' '21:17:53.009",' '"error":' '{' '"time":' '"2024-05-04T21:18:05.373668238+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449532047657270997 is earlier than or equal to GC safepoint at '449532047657271297"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365133008823964645, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449532047657271297, '"checkpoint_time":' '"2024-05-04' '21:17:53.009",' '"error":' '{' '"time":' '"2024-05-04T21:18:05.373668238+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449532047657270997 is earlier than or equal to GC safepoint at '449532047657271297"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449532047657270997 is earlier than or equal to GC safepoint at 449532047657271297' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449532047657270997 is earlier than or equal to GC safepoint at 449532047657271297 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6816.out cli changefeed resume -c changefeed-error PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 19-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f04c0780018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q, pid:8632, start at 2024-05-04 21:18:05.392016986 +0800 CST m=+5.265250195 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:05.399 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:05.392 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:05.392 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f04c2d80008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-jdj9-b880q, pid:8713, start at 2024-05-04 21:18:05.504399541 +0800 CST m=+5.323660103 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:05.511 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:05.494 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:05.494 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x table changefeed_error.usertable not exists for 1-th check, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.10042.out cli tso query --pd=http://127.0.0.1:2379 table changefeed_error.usertable exists check diff failed 1-th time, retry later check diff failed 3-th time, retry later + set +x + tso='449532052318978050 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532052318978050 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 21-th check, retry later ***************** properties ***************** "mysql.port"="4000" "mysql.user"="root" "readallfields"="true" "workload"="core" "requestdistribution"="uniform" "scanproportion"="0" "insertproportion"="0" "threadcount"="2" "recordcount"="10" "dotransactions"="false" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.db"="multi_capture_1" "operationcount"="0" "readproportion"="0" ********************************************** Run finished, takes 9.455044ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1776.9, Avg(us): 1811, Min(us): 1107, Max(us): 3734, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "readproportion"="0" "mysql.host"="127.0.0.1" "threadcount"="2" "dotransactions"="false" "requestdistribution"="uniform" "readallfields"="true" "workload"="core" "scanproportion"="0" "recordcount"="10" "updateproportion"="0" "operationcount"="0" "mysql.port"="4000" "mysql.user"="root" "mysql.db"="multi_capture_2" "insertproportion"="0" ********************************************** Run finished, takes 8.186835ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2152.6, Avg(us): 1549, Min(us): 932, Max(us): 3410, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "requestdistribution"="uniform" "mysql.db"="multi_capture_3" "workload"="core" "readallfields"="true" "recordcount"="10" "dotransactions"="false" "updateproportion"="0" "scanproportion"="0" "threadcount"="2" "insertproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "readproportion"="0" "mysql.port"="4000" "operationcount"="0" ********************************************** Run finished, takes 8.389612ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2122.1, Avg(us): 1602, Min(us): 1004, Max(us): 3572, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "operationcount"="0" "threadcount"="2" "mysql.db"="multi_capture_4" "readallfields"="true" "mysql.user"="root" "dotransactions"="false" "insertproportion"="0" "mysql.port"="4000" "recordcount"="10" "workload"="core" "updateproportion"="0" "requestdistribution"="uniform" "readproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" ********************************************** Run finished, takes 12.729505ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1200.5, Avg(us): 2275, Min(us): 1162, Max(us): 4529, 95th(us): 5000, 99th(us): 5000 [Sat May 4 21:18:13 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1016810170.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully ***************** properties ***************** "mysql.host"="127.0.0.1" "mysql.db"="changefeed_error" "readproportion"="0" "operationcount"="0" "updateproportion"="0" "scanproportion"="0" "mysql.user"="root" "insertproportion"="0" "readallfields"="true" "requestdistribution"="uniform" "threadcount"="4" "dotransactions"="false" "recordcount"="20" "workload"="core" "mysql.port"="4000" ********************************************** Run finished, takes 5.174684ms INSERT - Takes(s): 0.0, Count: 20, OPS: 6530.6, Avg(us): 948, Min(us): 470, Max(us): 2332, 95th(us): 3000, 99th(us): 3000 check diff successfully {"id":"1a83b53b-2650-4200-96cc-d04a9a5fcb08","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828675} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 + [[ /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 22-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 + [[ /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later check diff failed 1-th time, retry later check diff successfully table ddl_manager.finish_mark not exists for 23-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:16 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c343ca97-b1a2-4fef-b321-914e850c37ea {"id":"c343ca97-b1a2-4fef-b321-914e850c37ea","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c43e c343ca97-b1a2-4fef-b321-914e850c37ea /tidb/cdc/default/default/upstream/7365133077408296149 {"id":7365133077408296149,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c343ca97-b1a2-4fef-b321-914e850c37ea {"id":"c343ca97-b1a2-4fef-b321-914e850c37ea","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c43e c343ca97-b1a2-4fef-b321-914e850c37ea /tidb/cdc/default/default/upstream/7365133077408296149 {"id":7365133077408296149,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c343ca97-b1a2-4fef-b321-914e850c37ea {"id":"c343ca97-b1a2-4fef-b321-914e850c37ea","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c43e c343ca97-b1a2-4fef-b321-914e850c37ea /tidb/cdc/default/default/upstream/7365133077408296149 {"id":7365133077408296149,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 21:18:16 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1023110233.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc) 3730435584 bytes in 15.10 secs (247093022 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 24-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=d4bec7b3-0fe9-49d9-82d1-f0ad72e42097 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-lpbc3 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-lpbc3 pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: pulsar... <<================= [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=5f3e7c36-92e6-432b-ba05-1fc80ce14bb6 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-q7cdx GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-q7cdx pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: pulsar... <<================= [Sat May 4 21:18:18 CST 2024] <<<<<< skip test case clustered_index for pulsar! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=35420a2a-16f7-4f4b-9828-a26005ec171c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-7v1cj GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-7v1cj pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh check diff successfully + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=6e9844a2-cbe1-4795-89e1-780b080febe7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-r93qw GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-r93qw pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=28fb5273-bd0f-423f-b7f9-0c572eb3a229 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-xhmtq GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-xhmtq pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:19 GMT < Content-Length: 1273 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68063958-642c-48ab-9787-b400e4f80983 {"id":"68063958-642c-48ab-9787-b400e4f80983","address":"127.0.0.1:8302","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828696} /tidb/cdc/default/__cdc_meta__/capture/c343ca97-b1a2-4fef-b321-914e850c37ea {"id":"c343ca97-b1a2-4fef-b321-914e850c37ea","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c43e c343ca97-b1a2-4fef-b321-914e850c37ea /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c474 68063958-642c-48ab-9787-b400e4f80983 /tidb/cdc/default/default/upstream/7365133077408296149 {"id":7365133077408296149,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68063958-642c-48ab-9787-b400e4f80983 {"id":"68063958-642c-48ab-9787-b400e4f80983","address":"127.0.0.1:8302","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828696} /tidb/cdc/default/__cdc_meta__/capture/c343ca97-b1a2-4fef-b321-914e850c37ea {"id":"c343ca97-b1a2-4fef-b321-914e850c37ea","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c43e c343ca97-b1a2-4fef-b321-914e850c37ea /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c474 68063958-642c-48ab-9787-b400e4f80983 /tidb/cdc/default/default/upstream/7365133077408296149 {"id":7365133077408296149,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68063958-642c-48ab-9787-b400e4f80983 {"id":"68063958-642c-48ab-9787-b400e4f80983","address":"127.0.0.1:8302","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828696} /tidb/cdc/default/__cdc_meta__/capture/c343ca97-b1a2-4fef-b321-914e850c37ea {"id":"c343ca97-b1a2-4fef-b321-914e850c37ea","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c43e c343ca97-b1a2-4fef-b321-914e850c37ea /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c474 68063958-642c-48ab-9787-b400e4f80983 /tidb/cdc/default/default/upstream/7365133077408296149 {"id":7365133077408296149,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 21:18:19 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.1028810290.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 + [[ /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/1a83b53b-2650-4200-96cc-d04a9a5fcb08 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later wait process cdc.test exit for 1-th time... [Pipeline] sh check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 25-th check, retry later [Pipeline] sh wait process cdc.test exit for 2-th time... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=06f8ff00-7005-42c7-bfb0-3353550dfecb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-l51s9 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2 pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-l51s9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: pulsar... <<================= wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 21:18:21 CST 2024] <<<<<< run test case split_region success! >>>>>> [Pipeline] sh find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=6b076179-671a-4401-b473-7c90b6dd0566 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-kqgnr GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-kqgnr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: pulsar... <<================= [Pipeline] // node Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 3-th time, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh pulsar G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=69ee7a06-fd08-4dae-956a-19b455d90427 JENKINS_NODE_COOKIE=aea17098-1cb3-40c0-81d5-76b47a3a6d9d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_pulsar_test","buildid":"1786742824457211905","prowjobid":"69ee7a06-fd08-4dae-956a-19b455d90427","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"7f3a787edb079ab2f7719b4b79f735e1697cc211","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/7f3a787edb079ab2f7719b4b79f735e1697cc211","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/usr/java/openjdk-17/bin:/usr/local/pulsar/bin:/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_pulsar_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1535 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786742824457211905 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_pulsar_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/1535/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-xfvtz GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-1c247575f7a5458f3a8df09baf43395940069f8be33f18a8fa953544ffab3376 NODE_LABELS=pingcap_tiflow_pull_cdc_integration_pulsar_test_1535-xfvtz pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_pulsar_test/display/redirect BUILD_NUMBER=1535 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: pulsar... <<================= table ddl_manager.finish_mark not exists for 26-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:22 GMT < Content-Length: 1753 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4d66901a-d28a-4e4b-aa1b-a2f31ad7af1d {"id":"4d66901a-d28a-4e4b-aa1b-a2f31ad7af1d","address":"127.0.0.1:8303","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828699} /tidb/cdc/default/__cdc_meta__/capture/68063958-642c-48ab-9787-b400e4f80983 {"id":"68063958-642c-48ab-9787-b400e4f80983","address":"127.0.0.1:8302","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828696} /tidb/cdc/default/__cdc_meta__/capture/c343ca97-b1a2-4fef-b321-914e850c37ea {"id":"c343ca97-b1a2-4fef-b321-914e850c37ea","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c43e c343ca97-b1a2-4fef-b321-914e850c37ea /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c474 68063958-642c-48ab-9787-b400e4f80983 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c495 4d66901a-d28a-4e4b-aa1b-a2f31ad7af1d /tidb/cdc/default/default/upstream/7365133077408296149 {"id":7365133077408296149,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4d66901a-d28a-4e4b-aa1b-a2f31ad7af1d {"id":"4d66901a-d28a-4e4b-aa1b-a2f31ad7af1d","address":"127.0.0.1:8303","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828699} /tidb/cdc/default/__cdc_meta__/capture/68063958-642c-48ab-9787-b400e4f80983 {"id":"68063958-642c-48ab-9787-b400e4f80983","address":"127.0.0.1:8302","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828696} /tidb/cdc/default/__cdc_meta__/capture/c343ca97-b1a2-4fef-b321-914e850c37ea {"id":"c343ca97-b1a2-4fef-b321-914e850c37ea","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c43e c343ca97-b1a2-4fef-b321-914e850c37ea /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c474 68063958-642c-48ab-9787-b400e4f80983 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c495 4d66901a-d28a-4e4b-aa1b-a2f31ad7af1d /tidb/cdc/default/default/upstream/7365133077408296149 {"id":7365133077408296149,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4d66901a-d28a-4e4b-aa1b-a2f31ad7af1d {"id":"4d66901a-d28a-4e4b-aa1b-a2f31ad7af1d","address":"127.0.0.1:8303","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828699} /tidb/cdc/default/__cdc_meta__/capture/68063958-642c-48ab-9787-b400e4f80983 {"id":"68063958-642c-48ab-9787-b400e4f80983","address":"127.0.0.1:8302","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828696} /tidb/cdc/default/__cdc_meta__/capture/c343ca97-b1a2-4fef-b321-914e850c37ea {"id":"c343ca97-b1a2-4fef-b321-914e850c37ea","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828693} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c43e c343ca97-b1a2-4fef-b321-914e850c37ea /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c474 68063958-642c-48ab-9787-b400e4f80983 /tidb/cdc/default/__cdc_meta__/owner/22318f43c103c495 4d66901a-d28a-4e4b-aa1b-a2f31ad7af1d /tidb/cdc/default/default/upstream/7365133077408296149 {"id":7365133077408296149,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:18:22 CST 2024] <<<<<< START pulsar cluster in multi_capture case >>>>>>' [Sat May 4 21:18:22 CST 2024] <<<<<< START pulsar cluster in multi_capture case >>>>>> + workdir=/tmp/tidb_cdc_test/multi_capture + cluster_type=normal + cd /tmp/tidb_cdc_test/multi_capture + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/multi_capture/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/multi_capture/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/multi_capture/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/multi_capture/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/multi_capture/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/multi_capture/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/multi_capture/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/multi_capture/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/multi_capture/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/multi_capture/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/multi_capture/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/multi_capture/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:18:22 CST 2024] <<<<<< START pulsar cluster in normal mode in multi_capture case >>>>>>' [Sat May 4 21:18:22 CST 2024] <<<<<< START pulsar cluster in normal mode in multi_capture case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/multi_capture/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/multi_capture/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/multi_capture/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 27-th check, retry later check diff failed 4-th time, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: pulsar... <<================= The 1 times to try to start tidb cluster... check diff failed 5-th time, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 21:18:25 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.70797081.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 28-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 29-th check, retry later The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d5d210f-9fc4-4d8c-bda9-ebf3673ebe8e {"id":"1d5d210f-9fc4-4d8c-bda9-ebf3673ebe8e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828705} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0c04204 1d5d210f-9fc4-4d8c-bda9-ebf3673ebe8e /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365133008823964645,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-15128?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:18:05.264290639+08:00","start-ts":449532047657271297,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T21:18:26.214315374+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532056360452098} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449532052730019843,"min-table-barrier-ts":449532052991901703,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365133008823964645 {"id":7365133008823964645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d5d210f-9fc4-4d8c-bda9-ebf3673ebe8e {"id":"1d5d210f-9fc4-4d8c-bda9-ebf3673ebe8e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828705} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0c04204 1d5d210f-9fc4-4d8c-bda9-ebf3673ebe8e /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365133008823964645,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-15128?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:18:05.264290639+08:00","start-ts":449532047657271297,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T21:18:26.214315374+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532056360452098} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449532052730019843,"min-table-barrier-ts":449532052991901703,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365133008823964645 {"id":7365133008823964645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d5d210f-9fc4-4d8c-bda9-ebf3673ebe8e {"id":"1d5d210f-9fc4-4d8c-bda9-ebf3673ebe8e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828705} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0c04204 1d5d210f-9fc4-4d8c-bda9-ebf3673ebe8e /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365133008823964645,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-15128?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:18:05.264290639+08:00","start-ts":449532047657271297,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T21:18:26.214315374+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532056360452098} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449532052730019843,"min-table-barrier-ts":449532052991901703,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365133008823964645 {"id":7365133008823964645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365133008823964645, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449532052730019843, "checkpoint_time": "2024-05-04 21:18:12.360", "error": { "time": "2024-05-04T21:18:26.214315374+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365133008823964645, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449532052730019843, "checkpoint_time": "2024-05-04 21:18:12.360", "error": { "time": "2024-05-04T21:18:26.214315374+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365133008823964645, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449532052730019843, "checkpoint_time": "2024-05-04 21:18:12.360", "error": { "time": "2024-05-04T21:18:26.214315374+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365133008823964645, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449532052730019843, '"checkpoint_time":' '"2024-05-04' '21:18:12.360",' '"error":' '{' '"time":' '"2024-05-04T21:18:26.214315374+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365133008823964645, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449532052730019843, '"checkpoint_time":' '"2024-05-04' '21:18:12.360",' '"error":' '{' '"time":' '"2024-05-04T21:18:26.214315374+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7191.out cli changefeed remove -c changefeed-error ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449532052730019843 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-15128?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] } Cache not saved (inner-step execution failed) check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 30-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) public/default ++ date + echo '[Sat May 4 21:18:31 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:18:31 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.10760.out cli changefeed create --start-ts=449532052318978050 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-multi-capture-test-16070?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 31-th check, retry later check diff failed 2-th time, retry later wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: aa5d43df-8f18-4634-acbd-d8b240f4da4d Info: {"upstream_id":7365133077408296149,"namespace":"default","id":"aa5d43df-8f18-4634-acbd-d8b240f4da4d","sink_uri":"pulsar://127.0.0.1:6650/ticdc-multi-capture-test-16070?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:18:32.244540531+08:00","start_ts":449532052318978050,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532052318978050,"checkpoint_ts":449532052318978050,"checkpoint_time":"2024-05-04 21:18:10.792"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 21:18:32 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.72867288.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0663b40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q, pid:1421, start at 2024-05-04 21:18:32.182333258 +0800 CST m=+5.136445040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:32.192 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:32.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:32.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0663b40006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q, pid:1421, start at 2024-05-04 21:18:32.182333258 +0800 CST m=+5.136445040 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:32.192 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:32.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:32.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0664500005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-7v1c-rwp1q, pid:1503, start at 2024-05-04 21:18:32.215654912 +0800 CST m=+5.120442448 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:32.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:32.212 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:32.212 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Verifying downstream PD is started... + set +x [Sat May 4 21:18:33 CST 2024] <<<<<< START Pulsar consumer in multi_capture case >>>>>> table multi_capture_1.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 32-th check, retry later check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0675600004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q, pid:1355, start at 2024-05-04 21:18:33.310598254 +0800 CST m=+5.269599369 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:33.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:33.304 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:33.304 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0657e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f, pid:1421, start at 2024-05-04 21:18:31.421621999 +0800 CST m=+5.346594727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:31.429 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:31.418 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:31.418 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0657e80004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f, pid:1421, start at 2024-05-04 21:18:31.421621999 +0800 CST m=+5.346594727 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:31.429 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:31.418 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:31.418 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0659740014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-q7cd-20h3f, pid:1497, start at 2024-05-04 21:18:31.542480499 +0800 CST m=+5.417040879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:31.552 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:31.517 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:31.517 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f067aa80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp, pid:1359, start at 2024-05-04 21:18:33.66763546 +0800 CST m=+5.090395748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:33.674 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:33.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:33.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f067aa80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp, pid:1359, start at 2024-05-04 21:18:33.66763546 +0800 CST m=+5.090395748 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:33.674 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:33.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:33.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f067d940007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-lpbc-z7bmp, pid:1439, start at 2024-05-04 21:18:33.835373107 +0800 CST m=+5.207935522 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:33.844 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:33.829 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:33.829 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2823.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table multi_capture_1.usertable exists table multi_capture_2.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:35 GMT < Content-Length: 884 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68cddfe3-f010-4ce4-b735-ddd1b7a752ad {"id":"68cddfe3-f010-4ce4-b735-ddd1b7a752ad","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828713} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0c04261 68cddfe3-f010-4ce4-b735-ddd1b7a752ad /tidb/cdc/default/default/upstream/7365133008823964645 {"id":7365133008823964645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68cddfe3-f010-4ce4-b735-ddd1b7a752ad {"id":"68cddfe3-f010-4ce4-b735-ddd1b7a752ad","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828713} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0c04261 68cddfe3-f010-4ce4-b735-ddd1b7a752ad /tidb/cdc/default/default/upstream/7365133008823964645 {"id":7365133008823964645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/68cddfe3-f010-4ce4-b735-ddd1b7a752ad {"id":"68cddfe3-f010-4ce4-b735-ddd1b7a752ad","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828713} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f43c0c04261 68cddfe3-f010-4ce4-b735-ddd1b7a752ad /tidb/cdc/default/default/upstream/7365133008823964645 {"id":7365133008823964645,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7342.out cli changefeed create --start-ts=449532047657271297 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-sink-retry-test-15128?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-1 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0675600004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q, pid:1355, start at 2024-05-04 21:18:33.310598254 +0800 CST m=+5.269599369 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:33.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:33.304 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:33.304 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0674900013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xhmt-zrc0q, pid:1441, start at 2024-05-04 21:18:33.276366244 +0800 CST m=+5.176906508 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:33.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:33.252 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:33.252 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sat May 4 21:18:36 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.28132815.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365133008823964645,"namespace":"default","id":"changefeed-error-1","sink_uri":"pulsar://127.0.0.1:6650/ticdc-sink-retry-test-15128?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:18:36.375734399+08:00","start_ts":449532047657271297,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532047657271297,"checkpoint_ts":449532047657271297,"checkpoint_time":"2024-05-04 21:17:53.009"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 33-th check, retry later check diff failed 4-th time, retry later [Pipeline] // stage [Pipeline] // stage ++ date + echo '[Sat May 4 21:18:36 CST 2024] <<<<<< START pulsar cluster in capture_session_done_during_task case >>>>>>' [Sat May 4 21:18:36 CST 2024] <<<<<< START pulsar cluster in capture_session_done_during_task case >>>>>> + workdir=/tmp/tidb_cdc_test/capture_session_done_during_task + cluster_type=normal + cd /tmp/tidb_cdc_test/capture_session_done_during_task + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/capture_session_done_during_task/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/capture_session_done_during_task/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/capture_session_done_during_task/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/capture_session_done_during_task/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:18:36 CST 2024] <<<<<< START pulsar cluster in normal mode in capture_session_done_during_task case >>>>>>' [Sat May 4 21:18:36 CST 2024] <<<<<< START pulsar cluster in normal mode in capture_session_done_during_task case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/capture_session_done_during_task/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 [Pipeline] } [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449532058841382913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532058841382913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 21:18:37 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.28662868.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // podTemplate table multi_capture_2.usertable exists table multi_capture_3.usertable not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0693900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws, pid:1368, start at 2024-05-04 21:18:35.240416771 +0800 CST m=+5.656936657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:35.246 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:35.236 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:35.236 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0693900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws, pid:1368, start at 2024-05-04 21:18:35.240416771 +0800 CST m=+5.656936657 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:35.246 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:35.236 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:35.236 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f068c800015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-r93q-7hzws, pid:1449, start at 2024-05-04 21:18:34.821531906 +0800 CST m=+5.182208205 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:34.829 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:34.834 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:34.834 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table ddl_manager.finish_mark not exists for 34-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 577 0 --:--:-- --:--:-- --:--:-- 581 + info='{"state":"normal","resolved_ts":449532047657271297,"checkpoint_ts":449532047657271297}' + echo '{"state":"normal","resolved_ts":449532047657271297,"checkpoint_ts":449532047657271297}' {"state":"normal","resolved_ts":449532047657271297,"checkpoint_ts":449532047657271297} ++ echo '{"state":"normal","resolved_ts":449532047657271297,"checkpoint_ts":449532047657271297}' ++ jq -r .state + state=normal + [[ ! normal == \w\a\r\n\i\n\g ]] + echo 'changefeed state normal does not equal to warning' changefeed state normal does not equal to warning + exit 1 run task failed 1-th time, retry later ++ date + echo '[Sat May 4 21:18:38 CST 2024] <<<<<< START pulsar cluster in changefeed_finish case >>>>>>' [Sat May 4 21:18:38 CST 2024] <<<<<< START pulsar cluster in changefeed_finish case >>>>>> + workdir=/tmp/tidb_cdc_test/changefeed_finish + cluster_type=normal + cd /tmp/tidb_cdc_test/changefeed_finish + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/changefeed_finish/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/changefeed_finish/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/changefeed_finish/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/changefeed_finish/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/changefeed_finish/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/changefeed_finish/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/changefeed_finish/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/changefeed_finish/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/changefeed_finish/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/changefeed_finish/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:18:38 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_finish case >>>>>>' [Sat May 4 21:18:38 CST 2024] <<<<<< START pulsar cluster in normal mode in changefeed_finish case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/changefeed_finish/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/changefeed_finish/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/changefeed_finish/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 check diff successfully Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f06b8d80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs, pid:1445, start at 2024-05-04 21:18:37.661282353 +0800 CST m=+5.143157189 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:37.668 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:37.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:37.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f06b8d80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs, pid:1445, start at 2024-05-04 21:18:37.661282353 +0800 CST m=+5.143157189 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:37.668 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:37.672 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:37.672 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f06b8d80015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-kqgn-xzsrs, pid:1525, start at 2024-05-04 21:18:37.665342449 +0800 CST m=+5.078815965 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:37.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:37.673 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:37.673 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] // cache + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:39 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5187ff8-6e37-406b-ba89-2a257576c18e {"id":"a5187ff8-6e37-406b-ba89-2a257576c18e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c1710ccd a5187ff8-6e37-406b-ba89-2a257576c18e /tidb/cdc/default/default/upstream/7365133193938033584 {"id":7365133193938033584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5187ff8-6e37-406b-ba89-2a257576c18e {"id":"a5187ff8-6e37-406b-ba89-2a257576c18e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c1710ccd a5187ff8-6e37-406b-ba89-2a257576c18e /tidb/cdc/default/default/upstream/7365133193938033584 {"id":7365133193938033584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5187ff8-6e37-406b-ba89-2a257576c18e {"id":"a5187ff8-6e37-406b-ba89-2a257576c18e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c1710ccd a5187ff8-6e37-406b-ba89-2a257576c18e /tidb/cdc/default/default/upstream/7365133193938033584 {"id":7365133193938033584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:18:39 CST 2024] <<<<<< START pulsar cluster in processor_resolved_ts_fallback case >>>>>>' [Sat May 4 21:18:39 CST 2024] <<<<<< START pulsar cluster in processor_resolved_ts_fallback case >>>>>> + workdir=/tmp/tidb_cdc_test/processor_resolved_ts_fallback + cluster_type=normal + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/processor_resolved_ts_fallback/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/processor_resolved_ts_fallback/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/processor_resolved_ts_fallback/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:18:39 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_resolved_ts_fallback case >>>>>>' [Sat May 4 21:18:39 CST 2024] <<<<<< START pulsar cluster in normal mode in processor_resolved_ts_fallback case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 2269 0 --:--:-- --:--:-- --:--:-- 2279 + info='{"state":"warning","resolved_ts":449532048050487320,"checkpoint_ts":449532048050487320,"last_warning":{"time":"2024-05-04T21:18:38.600436605+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449532048050487320,"checkpoint_ts":449532048050487320,"last_warning":{"time":"2024-05-04T21:18:38.600436605+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449532048050487320,"checkpoint_ts":449532048050487320,"last_warning":{"time":"2024-05-04T21:18:38.600436605+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":449532048050487320,"checkpoint_ts":449532048050487320,"last_warning":{"time":"2024-05-04T21:18:38.600436605+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449532048050487320,"checkpoint_ts":449532048050487320,"last_warning":{"time":"2024-05-04T21:18:38.600436605+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.7403.out cli changefeed remove -c changefeed-error-1 [Sat May 4 21:18:39 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.28442846.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:40 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11464ee5-00d2-4082-98f3-cd0c5c962cb3 {"id":"11464ee5-00d2-4082-98f3-cd0c5c962cb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828717} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c172d1cf 11464ee5-00d2-4082-98f3-cd0c5c962cb3 /tidb/cdc/default/default/upstream/7365133194006741768 {"id":7365133194006741768,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11464ee5-00d2-4082-98f3-cd0c5c962cb3 {"id":"11464ee5-00d2-4082-98f3-cd0c5c962cb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828717} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c172d1cf 11464ee5-00d2-4082-98f3-cd0c5c962cb3 /tidb/cdc/default/default/upstream/7365133194006741768 {"id":7365133194006741768,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11464ee5-00d2-4082-98f3-cd0c5c962cb3 {"id":"11464ee5-00d2-4082-98f3-cd0c5c962cb3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828717} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c172d1cf 11464ee5-00d2-4082-98f3-cd0c5c962cb3 /tidb/cdc/default/default/upstream/7365133194006741768 {"id":7365133194006741768,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:18:40 CST 2024] <<<<<< START pulsar cluster in tiflash case >>>>>>' [Sat May 4 21:18:40 CST 2024] <<<<<< START pulsar cluster in tiflash case >>>>>> + workdir=/tmp/tidb_cdc_test/tiflash + cluster_type=mtls + cd /tmp/tidb_cdc_test/tiflash + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/tiflash/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/tiflash/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/tiflash/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/tiflash/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/tiflash/pulsar_standalone.conf + pulsar_port=6650 + '[' mtls == mtls ']' + pulsar_port=6651 + gen_mtls_config + openssl genrsa -out ca.key.pem 2048 Generating RSA private key, 2048 bit long modulus ...................................................................................................+++ ............................+++ e is 65537 (0x10001) + openssl req -x509 -new -nodes -key ca.key.pem -subj /CN=CARoot -days 365 -out ca.cert.pem + openssl genrsa -out server.key.pem 2048 Generating RSA private key, 2048 bit long modulus ......+++ ...........+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in server.key.pem -out server.key-pk8.pem -nocrypt + echo '[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + openssl req -new -config server.conf -key server.key.pem -out server.csr.pem -sha256 + openssl x509 -req -in server.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out server.cert.pem -days 365 -extensions v3_ext -extfile server.conf -sha256 Signature ok subject=/CN=server Getting CA Private Key + openssl genrsa -out broker_client.key.pem 2048 Generating RSA private key, 2048 bit long modulus ..........................+++ ...........................................................................+++ e is 65537 (0x10001) + openssl pkcs8 -topk8 -inform PEM -outform PEM -in broker_client.key.pem -out broker_client.key-pk8.pem -nocrypt + openssl req -new -subj /CN=broker_client -key broker_client.key.pem -out broker_client.csr.pem -sha256 + openssl x509 -req -in broker_client.csr.pem -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial -out broker_client.cert.pem -days 365 -sha256 Signature ok subject=/CN=broker_client Getting CA Private Key + echo ' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/tiflash/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/tiflash/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' + echo ' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/tiflash/ca.cert.pem' [Pipeline] } check diff failed 1-th time, retry later [Pipeline] // container ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 35-th check, retry later [Pipeline] // container [Pipeline] } [Pipeline] } ++ date + echo '[Sat May 4 21:18:40 CST 2024] <<<<<< START pulsar cluster in mtls mode in tiflash case >>>>>>' [Sat May 4 21:18:40 CST 2024] <<<<<< START pulsar cluster in mtls mode in tiflash case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6651 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/tiflash/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/tiflash/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/tiflash/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Pipeline] // withEnv [Pipeline] } [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_pulsar_test-1535/tiflow-cdc already exists) Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449532048050487320 SinkURI: pulsar://127.0.0.1:6650/ticdc-sink-retry-test-15128?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Pipeline] // dir VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f06e0b4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz, pid:1484, start at 2024-05-04 21:18:40.183707868 +0800 CST m=+5.219785502 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:40.190 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:40.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:40.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Post stage [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Pipeline] // stage [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } table multi_capture_4.usertable not exists for 2-th check, retry later [Pipeline] // dir Post stage [Pipeline] sh check diff failed 2-th time, retry later [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } table ddl_manager.finish_mark not exists for 36-th check, retry later [Pipeline] sh + set +x + nc -z localhost 6651 + i=2 + '[' 2 -gt 20 ']' + sleep 2 + ls /tmp/tidb_cdc_test/ cov.row_format.cli.2821.out cov.row_format.cli.3324.out row_format sql_res.row_format.txt ++ find /tmp/tidb_cdc_test/ -type f -name '*.log' + tar -cvzf log-G03.tar.gz /tmp/tidb_cdc_test/row_format/tikv2.log /tmp/tidb_cdc_test/row_format/stdout.log /tmp/tidb_cdc_test/row_format/down_pd.log /tmp/tidb_cdc_test/row_format/tiflash/log/error.log /tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log /tmp/tidb_cdc_test/row_format/tiflash/log/server.log /tmp/tidb_cdc_test/row_format/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/row_format/pd1/region-meta/000001.log /tmp/tidb_cdc_test/row_format/pd1/hot-region/000001.log /tmp/tidb_cdc_test/row_format/tidb.log /tmp/tidb_cdc_test/row_format/pulsar_stdout.log /tmp/tidb_cdc_test/row_format/tikv2/db/000005.log /tmp/tidb_cdc_test/row_format/pd1.log /tmp/tidb_cdc_test/row_format/cdc_pulsar_consumer_stdout.log /tmp/tidb_cdc_test/row_format/tidb_down.log /tmp/tidb_cdc_test/row_format/cdc.log /tmp/tidb_cdc_test/row_format/tikv1.log /tmp/tidb_cdc_test/row_format/tikv_down/db/000005.log /tmp/tidb_cdc_test/row_format/tidb-slow.log /tmp/tidb_cdc_test/row_format/tikv3.log /tmp/tidb_cdc_test/row_format/tikv1/db/000005.log /tmp/tidb_cdc_test/row_format/cdc_pulsar_consumer.log /tmp/tidb_cdc_test/row_format/pulsar-metadata/000004.log /tmp/tidb_cdc_test/row_format/tikv_down.log /tmp/tidb_cdc_test/row_format/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/row_format/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/row_format/pulsar-bookie/current/0.log /tmp/tidb_cdc_test/row_format/pulsar-bookie/current/1.log /tmp/tidb_cdc_test/row_format/pulsar-bookie/current/ledgers/000004.log /tmp/tidb_cdc_test/row_format/pulsar-bookie/current/locations/000004.log /tmp/tidb_cdc_test/row_format/tidb_other.log /tmp/tidb_cdc_test/row_format/tikv3/db/000005.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0002/000002.log tar: Removing leading `/' from member names /tmp/tidb_cdc_test/row_format/tikv2.log /tmp/tidb_cdc_test/row_format/stdout.log /tmp/tidb_cdc_test/row_format/down_pd.log /tmp/tidb_cdc_test/row_format/tiflash/log/error.log /tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log /tmp/tidb_cdc_test/row_format/tiflash/log/server.log /tmp/tidb_cdc_test/row_format/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/row_format/pd1/region-meta/000001.log /tmp/tidb_cdc_test/row_format/pd1/hot-region/000001.log /tmp/tidb_cdc_test/row_format/tidb.log /tmp/tidb_cdc_test/row_format/pulsar_stdout.log /tmp/tidb_cdc_test/row_format/tikv2/db/000005.log /tmp/tidb_cdc_test/row_format/pd1.log /tmp/tidb_cdc_test/row_format/cdc_pulsar_consumer_stdout.log /tmp/tidb_cdc_test/row_format/tidb_down.log /tmp/tidb_cdc_test/row_format/cdc.log /tmp/tidb_cdc_test/row_format/tikv1.log + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } + ls /tmp/tidb_cdc_test/ cov.kv_client_stream_reconnect.33413343.out cov.split_region.65456547.out cov.split_region.cli.6497.out cov.split_region.cli.7029.out kv_client_stream_reconnect split_region sql_res.kv_client_stream_reconnect.txt sql_res.split_region.txt ++ find /tmp/tidb_cdc_test/ -type f -name '*.log' + tar -cvzf log-G07.tar.gz /tmp/tidb_cdc_test/split_region/tiflash/log/error.log /tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log /tmp/tidb_cdc_test/split_region/tiflash/log/server.log /tmp/tidb_cdc_test/split_region/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/split_region/pulsar_stdout.log /tmp/tidb_cdc_test/split_region/tikv1.log /tmp/tidb_cdc_test/split_region/tidb_down.log /tmp/tidb_cdc_test/split_region/cdc_pulsar_consumer.log /tmp/tidb_cdc_test/split_region/tidb.log /tmp/tidb_cdc_test/split_region/stdout.log /tmp/tidb_cdc_test/split_region/tikv1/db/000005.log /tmp/tidb_cdc_test/split_region/sync_diff_inspector.log /tmp/tidb_cdc_test/split_region/tidb-slow.log /tmp/tidb_cdc_test/split_region/tikv2/db/000005.log /tmp/tidb_cdc_test/split_region/cdc_pulsar_consumer_stdout.log /tmp/tidb_cdc_test/split_region/pulsar-bookie/current/locations/000004.log /tmp/tidb_cdc_test/split_region/pulsar-bookie/current/ledgers/000004.log /tmp/tidb_cdc_test/split_region/tikv3/db/000005.log /tmp/tidb_cdc_test/split_region/pd1/hot-region/000001.log /tmp/tidb_cdc_test/split_region/pd1/region-meta/000001.log /tmp/tidb_cdc_test/split_region/pd1.log /tmp/tidb_cdc_test/split_region/cdc.log /tmp/tidb_cdc_test/split_region/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/split_region/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/split_region/down_pd.log /tmp/tidb_cdc_test/split_region/tikv2.log /tmp/tidb_cdc_test/split_region/tikv_down.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/split_region/pulsar-metadata/000004.log /tmp/tidb_cdc_test/split_region/oauth_server.log /tmp/tidb_cdc_test/split_region/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/split_region/tikv_down/db/000005.log /tmp/tidb_cdc_test/split_region/tikv3.log /tmp/tidb_cdc_test/split_region/tidb_other.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_stdout.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv1.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb_down.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_pulsar_consumer.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/stdout.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/sync_diff_inspector.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb-slow.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_pulsar_consumer_stdout.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/pd1.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/down_pd.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv2.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv_down.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/oauth_server.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv3.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb_other.log tar: Removing leading `/' from member names /tmp/tidb_cdc_test/split_region/tiflash/log/error.log /tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log /tmp/tidb_cdc_test/split_region/tiflash/log/server.log /tmp/tidb_cdc_test/split_region/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/split_region/pulsar_stdout.log /tmp/tidb_cdc_test/split_region/tikv1.log /tmp/tidb_cdc_test/split_region/tidb_down.log /tmp/tidb_cdc_test/split_region/cdc_pulsar_consumer.log /tmp/tidb_cdc_test/split_region/tidb.log /tmp/tidb_cdc_test/split_region/stdout.log /tmp/tidb_cdc_test/split_region/tikv1/db/000005.log /tmp/tidb_cdc_test/split_region/sync_diff_inspector.log /tmp/tidb_cdc_test/split_region/tidb-slow.log /tmp/tidb_cdc_test/split_region/tikv2/db/000005.log /tmp/tidb_cdc_test/split_region/cdc_pulsar_consumer_stdout.log /tmp/tidb_cdc_test/split_region/pulsar-bookie/current/locations/000004.log /tmp/tidb_cdc_test/split_region/pulsar-bookie/current/ledgers/000004.log /tmp/tidb_cdc_test/split_region/tikv3/db/000005.log /tmp/tidb_cdc_test/split_region/pd1/hot-region/000001.log /tmp/tidb_cdc_test/split_region/pd1/region-meta/000001.log /tmp/tidb_cdc_test/split_region/pd1.log /tmp/tidb_cdc_test/split_region/cdc.log /tmp/tidb_cdc_test/split_region/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/split_region/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/split_region/down_pd.log /tmp/tidb_cdc_test/split_region/tikv2.log /tmp/tidb_cdc_test/split_region/tikv_down.log < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:42 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/688d8886-5f01-47a7-b688-d395dce951ad {"id":"688d8886-5f01-47a7-b688-d395dce951ad","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828720} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c17ec8cc 688d8886-5f01-47a7-b688-d395dce951ad /tidb/cdc/default/default/upstream/7365133211649941112 {"id":7365133211649941112,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/688d8886-5f01-47a7-b688-d395dce951ad {"id":"688d8886-5f01-47a7-b688-d395dce951ad","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828720} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c17ec8cc 688d8886-5f01-47a7-b688-d395dce951ad /tidb/cdc/default/default/upstream/7365133211649941112 {"id":7365133211649941112,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/688d8886-5f01-47a7-b688-d395dce951ad {"id":"688d8886-5f01-47a7-b688-d395dce951ad","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828720} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c17ec8cc 688d8886-5f01-47a7-b688-d395dce951ad /tidb/cdc/default/default/upstream/7365133211649941112 {"id":7365133211649941112,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully ++ date + echo '[Sat May 4 21:18:43 CST 2024] <<<<<< START pulsar cluster in owner_resign case >>>>>>' [Sat May 4 21:18:43 CST 2024] <<<<<< START pulsar cluster in owner_resign case >>>>>> + workdir=/tmp/tidb_cdc_test/owner_resign + cluster_type=normal + cd /tmp/tidb_cdc_test/owner_resign + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/owner_resign/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/owner_resign/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/owner_resign/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/owner_resign/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/owner_resign/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/owner_resign/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/owner_resign/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/owner_resign/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/owner_resign/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/owner_resign/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:18:43 CST 2024] <<<<<< START pulsar cluster in normal mode in owner_resign case >>>>>>' [Sat May 4 21:18:43 CST 2024] <<<<<< START pulsar cluster in normal mode in owner_resign case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/owner_resign/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/owner_resign/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/owner_resign/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 ++ date + echo '[Sat May 4 21:18:43 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:18:43 CST 2024] <<<<<< pulsar is ready >>>>>> wait process cdc.test exit for 1-th time... /tmp/tidb_cdc_test/row_format/tikv_down/db/000005.log /tmp/tidb_cdc_test/row_format/tidb-slow.log /tmp/tidb_cdc_test/row_format/tikv3.log /tmp/tidb_cdc_test/row_format/tikv1/db/000005.log /tmp/tidb_cdc_test/row_format/cdc_pulsar_consumer.log /tmp/tidb_cdc_test/row_format/pulsar-metadata/000004.log /tmp/tidb_cdc_test/row_format/tikv_down.log [Pipeline] // stage [Pipeline] // stage VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f06e0b4000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz, pid:1484, start at 2024-05-04 21:18:40.183707868 +0800 CST m=+5.219785502 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:40.190 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:40.173 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:40.173 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f06e1ac0004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-xfvt-xj0wz, pid:1575, start at 2024-05-04 21:18:40.238573332 +0800 CST m=+5.215475321 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:40.245 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:40.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:40.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0002/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/split_region/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/split_region/pulsar-metadata/000004.log /tmp/tidb_cdc_test/split_region/oauth_server.log /tmp/tidb_cdc_test/split_region/sync_diff/output/sync_diff.log /tmp/tidb_cdc_test/split_region/tikv_down/db/000005.log /tmp/tidb_cdc_test/split_region/tikv3.log [Pipeline] } [Pipeline] } /tmp/tidb_cdc_test/row_format/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/row_format/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/row_format/pulsar-bookie/current/0.log /tmp/tidb_cdc_test/row_format/pulsar-bookie/current/1.log /tmp/tidb_cdc_test/row_format/pulsar-bookie/current/ledgers/000004.log /tmp/tidb_cdc_test/row_format/pulsar-bookie/current/locations/000004.log /tmp/tidb_cdc_test/row_format/tidb_other.log /tmp/tidb_cdc_test/row_format/tikv3/db/000005.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0000/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0001/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0006/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0004/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0007/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0005/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0003/000002.log /tmp/tidb_cdc_test/row_format/cdc_data/tmp/sorter/0002/000002.log + ls -alh log-G03.tar.gz -rw-r--r--. 1 jenkins jenkins 5.0M May 4 21:18 log-G03.tar.gz + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Pipeline] archiveArtifacts Archiving artifacts + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.3296.out cli tso query --pd=http://127.0.0.1:2379 wait process cdc.test exit for 2-th time... /tmp/tidb_cdc_test/split_region/tidb_other.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/pulsar_stdout.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv1.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb_down.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_pulsar_consumer.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/stdout.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/sync_diff_inspector.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb-slow.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_pulsar_consumer_stdout.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/pd1.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/down_pd.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv2.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv_down.log table multi_capture_4.usertable exists check diff failed 1-th time, retry later /tmp/tidb_cdc_test/kv_client_stream_reconnect/oauth_server.log /tmp/tidb_cdc_test/kv_client_stream_reconnect/tikv3.log check diff failed 3-th time, retry later Recording fingerprints cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 21:18:44 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.74677469.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0712c40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2, pid:1380, start at 2024-05-04 21:18:43.410950526 +0800 CST m=+5.147834508 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:43.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:43.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:43.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f0712c40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2, pid:1380, start at 2024-05-04 21:18:43.410950526 +0800 CST m=+5.147834508 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:43.417 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:43.377 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:43.377 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0f071440000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-pulsar-test-1535-l51s-j43f2, pid:1469, start at 2024-05-04 21:18:43.489660696 +0800 CST m=+5.169607091 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-21:20:43.496 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-21:18:43.472 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-21:08:43.472 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table ddl_manager.finish_mark not exists for 37-th check, retry later [Pipeline] // timeout /tmp/tidb_cdc_test/kv_client_stream_reconnect/tidb_other.log + ls -alh log-G07.tar.gz -rw-r--r-- 1 jenkins jenkins 7.1M May 4 21:18 log-G07.tar.gz [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] archiveArtifacts Archiving artifacts Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } public/default ++ date + echo '[Sat May 4 21:18:44 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:18:44 CST 2024] <<<<<< pulsar is ready >>>>>> [Sat May 4 21:18:44 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.32283230.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv + nc -z localhost 6651 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2915.out cli tso query --pd=http://127.0.0.1:2379 [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G03' Recording fingerprints Sending interrupt signal to process Killing processes [Sat May 4 21:18:44 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.29402942.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449532060982837249 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532060982837249 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 21:18:45 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.33313333.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 kill finished with exit code 0 Sending interrupt signal to process Killing processes + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 kill finished with exit code 0 Sending interrupt signal to process Killing processes public/default ++ date + echo '[Sat May 4 21:18:45 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:18:45 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.3282.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-19193?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714828726.245772,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0023861c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes check diff successfully ***************** properties ***************** "insertproportion"="0" "scanproportion"="0" "readallfields"="true" "dotransactions"="false" "threadcount"="2" "workload"="core" "mysql.host"="127.0.0.1" "recordcount"="20" "requestdistribution"="uniform" "readproportion"="0" "mysql.user"="root" "mysql.port"="4000" "mysql.db"="multi_capture_1" "updateproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 22.502963ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1419.4, Avg(us): 2195, Min(us): 608, Max(us): 8373, 95th(us): 9000, 99th(us): 9000 ***************** properties ***************** "mysql.user"="root" "requestdistribution"="uniform" "insertproportion"="0" "mysql.db"="multi_capture_2" "mysql.host"="127.0.0.1" "recordcount"="20" "readallfields"="true" "workload"="core" "operationcount"="0" "mysql.port"="4000" "scanproportion"="0" "dotransactions"="false" "threadcount"="2" "readproportion"="0" "updateproportion"="0" ********************************************** Run finished, takes 10.826694ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2074.1, Avg(us): 1025, Min(us): 607, Max(us): 1514, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.user"="root" "mysql.db"="multi_capture_3" "threadcount"="2" "recordcount"="20" "mysql.port"="4000" "readallfields"="true" "workload"="core" "dotransactions"="false" "operationcount"="0" "updateproportion"="0" "readproportion"="0" "insertproportion"="0" "scanproportion"="0" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" ********************************************** Run finished, takes 12.725952ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1741.2, Avg(us): 1087, Min(us): 589, Max(us): 1946, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "workload"="core" "dotransactions"="false" "updateproportion"="0" "mysql.user"="root" "threadcount"="2" "recordcount"="20" "readallfields"="true" "requestdistribution"="uniform" "operationcount"="0" "readproportion"="0" "mysql.db"="multi_capture_4" "mysql.host"="127.0.0.1" "mysql.port"="4000" "scanproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 10.758988ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2100.6, Avg(us): 1019, Min(us): 502, Max(us): 1791, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later {"level":"warn","ts":1714828726.5176458,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003fece00/127.0.0.1:2479","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714828726.517903,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022d0540/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2923.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 4-th time, retry later Create changefeed successfully! ID: 9a4d152b-6fdc-4456-b817-c0b5c3a1e821 Info: {"upstream_id":7365133193938033584,"namespace":"default","id":"9a4d152b-6fdc-4456-b817-c0b5c3a1e821","sink_uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-19193?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:18:46.873350592+08:00","start_ts":449532061619060738,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532061619060738,"checkpoint_ts":449532061619060738,"checkpoint_time":"2024-05-04 21:18:46.269"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + tso='449532061449977857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532061449977857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 21:18:47 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.29662968.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:47 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eed57a20-5439-4bdb-b0b1-7df813eb1292 {"id":"eed57a20-5439-4bdb-b0b1-7df813eb1292","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828724} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c18c3626 eed57a20-5439-4bdb-b0b1-7df813eb1292 /tidb/cdc/default/default/upstream/7365133226535938941 {"id":7365133226535938941,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eed57a20-5439-4bdb-b0b1-7df813eb1292 {"id":"eed57a20-5439-4bdb-b0b1-7df813eb1292","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828724} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c18c3626 eed57a20-5439-4bdb-b0b1-7df813eb1292 /tidb/cdc/default/default/upstream/7365133226535938941 + grep -q 'failed to get info:' {"id":7365133226535938941,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/eed57a20-5439-4bdb-b0b1-7df813eb1292 {"id":"eed57a20-5439-4bdb-b0b1-7df813eb1292","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828724} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c18c3626 eed57a20-5439-4bdb-b0b1-7df813eb1292 /tidb/cdc/default/default/upstream/7365133226535938941 {"id":7365133226535938941,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:18:47 CST 2024] <<<<<< START pulsar cluster in batch_add_table case >>>>>>' [Sat May 4 21:18:47 CST 2024] <<<<<< START pulsar cluster in batch_add_table case >>>>>> + workdir=/tmp/tidb_cdc_test/batch_add_table + cluster_type=normal + cd /tmp/tidb_cdc_test/batch_add_table + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/batch_add_table/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/batch_add_table/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/batch_add_table/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/batch_add_table/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/batch_add_table/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:18:47 CST 2024] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>>' [Sat May 4 21:18:47 CST 2024] <<<<<< START pulsar cluster in normal mode in batch_add_table case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/batch_add_table/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/batch_add_table/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:47 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15fcac75-1d84-49ed-906e-f3d936933bd7 {"id":"15fcac75-1d84-49ed-906e-f3d936933bd7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828725} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c174e3e6 15fcac75-1d84-49ed-906e-f3d936933bd7 /tidb/cdc/default/default/upstream/7365133204191221815 {"id":7365133204191221815,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15fcac75-1d84-49ed-906e-f3d936933bd7 {"id":"15fcac75-1d84-49ed-906e-f3d936933bd7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828725} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c174e3e6 15fcac75-1d84-49ed-906e-f3d936933bd7 /tidb/cdc/default/default/upstream/7365133204191221815 {"id":7365133204191221815,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15fcac75-1d84-49ed-906e-f3d936933bd7 {"id":"15fcac75-1d84-49ed-906e-f3d936933bd7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828725} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c174e3e6 15fcac75-1d84-49ed-906e-f3d936933bd7 /tidb/cdc/default/default/upstream/7365133204191221815 {"id":7365133204191221815,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714828728.1419866,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00209bc00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 + set +x [Sat May 4 21:18:48 CST 2024] <<<<<< START Pulsar consumer in processor_resolved_ts_fallback case >>>>>> public/default ++ date + echo '[Sat May 4 21:18:48 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:18:48 CST 2024] <<<<<< pulsar is ready >>>>>> + set +x + tso='449532061778182145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449532061778182145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 21:18:48 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.29612963.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo '' + '[' 0 -eq 50 ']' + sleep 3 [Sat May 4 21:18:48 CST 2024] <<<<<< START Pulsar consumer in changefeed_finish case >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:48 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c07b86d-5ec3-4975-bc54-ac47f1e51375 {"id":"6c07b86d-5ec3-4975-bc54-ac47f1e51375","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828725} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c17ae709 6c07b86d-5ec3-4975-bc54-ac47f1e51375 /tidb/cdc/default/default/upstream/7365133206273819617 {"id":7365133206273819617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c07b86d-5ec3-4975-bc54-ac47f1e51375 {"id":"6c07b86d-5ec3-4975-bc54-ac47f1e51375","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828725} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c17ae709 6c07b86d-5ec3-4975-bc54-ac47f1e51375 /tidb/cdc/default/default/upstream/7365133206273819617 {"id":7365133206273819617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6c07b86d-5ec3-4975-bc54-ac47f1e51375 {"id":"6c07b86d-5ec3-4975-bc54-ac47f1e51375","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828725} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c17ae709 6c07b86d-5ec3-4975-bc54-ac47f1e51375 /tidb/cdc/default/default/upstream/7365133206273819617 {"id":7365133206273819617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 1-th time, retry later Create changefeed successfully! ID: 1e65e02f-f640-48dc-abd9-d90182006fff Info: {"upstream_id":7365133194006741768,"namespace":"default","id":"1e65e02f-f640-48dc-abd9-d90182006fff","sink_uri":"pulsar+ssl://127.0.0.1:6651/ticdc-tiflash-test-22751?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:18:49.055356482+08:00","start_ts":449532058841382913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"tls-trust-certs-file-path":"/tmp/tidb_cdc_test/tiflash/ca.cert.pem","connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30,"auth-tls-certificate-path":"/tmp/tidb_cdc_test/tiflash/broker_client.cert.pem","auth-tls-private-key-path":"/tmp/tidb_cdc_test/tiflash/broker_client.key-pk8.pem"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532058841382913,"checkpoint_ts":449532058841382913,"checkpoint_time":"2024-05-04 21:18:35.673"} [Sat May 4 21:18:49 CST 2024] <<<<<< START Pulsar consumer in tiflash case >>>>>> cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later + nc -z localhost 6650 + i=2 + '[' 2 -gt 20 ']' + sleep 2 [Sat May 4 21:18:50 CST 2024] <<<<<< START Pulsar consumer in capture_session_done_during_task case >>>>>> lease 22318f43c17ae709 revoked table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later table capture_session_done_during_task.t exists check diff failed 1-th time, retry later table processor_resolved_ts_fallback.t1 exists [Sat May 4 21:18:51 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.34393441.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:51 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41fb6bbf-9e68-4448-9715-24f5f7b7036c {"id":"41fb6bbf-9e68-4448-9715-24f5f7b7036c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828728} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c195cbd7 41fb6bbf-9e68-4448-9715-24f5f7b7036c /tidb/cdc/default/default/upstream/7365133237132038164 {"id":7365133237132038164,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41fb6bbf-9e68-4448-9715-24f5f7b7036c {"id":"41fb6bbf-9e68-4448-9715-24f5f7b7036c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828728} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c195cbd7 41fb6bbf-9e68-4448-9715-24f5f7b7036c /tidb/cdc/default/default/upstream/7365133237132038164 {"id":7365133237132038164,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/41fb6bbf-9e68-4448-9715-24f5f7b7036c {"id":"41fb6bbf-9e68-4448-9715-24f5f7b7036c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828728} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c195cbd7 41fb6bbf-9e68-4448-9715-24f5f7b7036c /tidb/cdc/default/default/upstream/7365133237132038164 {"id":7365133237132038164,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:18:51 CST 2024] <<<<<< START pulsar cluster in resolve_lock case >>>>>>' [Sat May 4 21:18:51 CST 2024] <<<<<< START pulsar cluster in resolve_lock case >>>>>> + workdir=/tmp/tidb_cdc_test/resolve_lock + cluster_type=normal + cd /tmp/tidb_cdc_test/resolve_lock + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/resolve_lock/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/resolve_lock/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/resolve_lock/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/resolve_lock/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' ++ cat + normal_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' ++ cat + mtls_client_conf=' webServiceUrl=https://localhost:8443/ brokerServiceUrl=pulsar+ssl://localhost:6651/ authPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls authParams=tlsCertFile:/tmp/tidb_cdc_test/resolve_lock/broker_client.cert.pem,tlsKeyFile:/tmp/tidb_cdc_test/resolve_lock/broker_client.key-pk8.pem tlsTrustCertsFilePath=/tmp/tidb_cdc_test/resolve_lock/ca.cert.pem' ++ cat + oauth_client_conf=' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/ authPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 authParams={"privateKey":"/tmp/tidb_cdc_test/resolve_lock/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"}' ++ cat + oauth_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderToken brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.oauth2.AuthenticationOAuth2 brokerClientAuthenticationParameters={"privateKey":"file:///tmp/tidb_cdc_test/resolve_lock/credential.json","audience":"cdc-api-uri","issuerUrl":"http://localhost:9096"} tokenSecretKey=data:;base64,U0poWDM2X0thcFlTeWJCdEpxMzVseFhfQnJyNExSVVJTa203UW1YSkdteThwVUZXOUVJT2NWUVBzeWt6OS1qag==' ++ cat + credential_json=' { "client_id":"1234", "client_secret":"e0KVlA2EiBfjoN13olyZd2kv1KL", "audience":"cdc-api-uri", "issuer_url":"http://localhost:9096", "type": "client_credentials" }' ++ cat + cert_server_conf='[ req ] default_bits = 2048 prompt = no default_md = sha256 distinguished_name = dn [ v3_ext ] authorityKeyIdentifier=keyid,issuer:always basicConstraints=CA:FALSE keyUsage=critical, digitalSignature, keyEncipherment extendedKeyUsage=serverAuth subjectAltName=@alt_names [ dn ] CN = server [ alt_names ] DNS.1 = localhost IP.1 = 127.0.0.1' + echo ' webServiceUrl=http://localhost:8080/ brokerServiceUrl=pulsar://localhost:6650/' + cp /usr/local/pulsar/conf/standalone.conf /tmp/tidb_cdc_test/resolve_lock/pulsar_standalone.conf + pulsar_port=6650 + '[' normal == mtls ']' + '[' normal == oauth ']' + echo 'no cluster type specified, using default configuration.' no cluster type specified, using default configuration. ++ date + echo '[Sat May 4 21:18:51 CST 2024] <<<<<< START pulsar cluster in normal mode in resolve_lock case >>>>>>' [Sat May 4 21:18:51 CST 2024] <<<<<< START pulsar cluster in normal mode in resolve_lock case >>>>>> + echo 'Waiting for pulsar port to be ready...' Waiting for pulsar port to be ready... + i=0 + nc -z localhost 6650 + /usr/local/pulsar/bin/pulsar standalone --config /tmp/tidb_cdc_test/resolve_lock/pulsar_standalone.conf -nfw --metadata-dir /tmp/tidb_cdc_test/resolve_lock/pulsar-metadata --bookkeeper-dir /tmp/tidb_cdc_test/resolve_lock/pulsar-bookie + i=1 + '[' 1 -gt 20 ']' + sleep 2 + nc -z localhost 6650 + echo 'Waiting for pulsar namespace to be ready...' Waiting for pulsar namespace to be ready... + i=0 + /usr/local/pulsar/bin/pulsar-admin namespaces list public kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714828727.7047243,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002291880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} script returned exit code 143 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:51 GMT < Content-Length: 816 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8bacddc5-3d05-4500-ac23-9097e8d66406 {"id":"8bacddc5-3d05-4500-ac23-9097e8d66406","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828727} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c18df7ed 8bacddc5-3d05-4500-ac23-9097e8d66406 /tidb/cdc/default/default/upstream/7365133235954936845 {"id":7365133235954936845,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8bacddc5-3d05-4500-ac23-9097e8d66406 {"id":"8bacddc5-3d05-4500-ac23-9097e8d66406","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828727} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c18df7ed 8bacddc5-3d05-4500-ac23-9097e8d66406 /tidb/cdc/default/default/upstream/7365133235954936845 {"id":7365133235954936845,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8bacddc5-3d05-4500-ac23-9097e8d66406 {"id":"8bacddc5-3d05-4500-ac23-9097e8d66406","address":"127.0.0.1:8300","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828727} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c18df7ed 8bacddc5-3d05-4500-ac23-9097e8d66406 /tidb/cdc/default/default/upstream/7365133235954936845 {"id":7365133235954936845,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x ++ date + echo '[Sat May 4 21:18:51 CST 2024] <<<<<< START pulsar cluster in simple case >>>>>>' [Sat May 4 21:18:51 CST 2024] <<<<<< START pulsar cluster in simple case >>>>>> + workdir=/tmp/tidb_cdc_test/simple + cluster_type=normal + cd /tmp/tidb_cdc_test/simple + DEFAULT_PULSAR_HOME=/usr/local/pulsar + pulsar_dir=/usr/local/pulsar ++ cat + mtls_conf=' authenticationEnabled=true authenticationProviders=org.apache.pulsar.broker.authentication.AuthenticationProviderTls brokerClientTlsEnabled=true brokerClientTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem brokerClientAuthenticationPlugin=org.apache.pulsar.client.impl.auth.AuthenticationTls brokerClientAuthenticationParameters={"tlsCertFile":"/tmp/tidb_cdc_test/simple/broker_client.cert.pem","tlsKeyFile":"/tmp/tidb_cdc_test/simple/broker_client.key-pk8.pem"} brokerServicePortTls=6651 webServicePortTls=8443 tlsTrustCertsFilePath=/tmp/tidb_cdc_test/simple/ca.cert.pem tlsCertificateFilePath=/tmp/tidb_cdc_test/simple/server.cert.pem tlsKeyFilePath=/tmp/tidb_cdc_test/simple/server.key-pk8.pem tlsRequireTrustedClientCertOnConnect=true tlsAllowInsecureConnection=false tlsCertRefreshCheckDurationSec=300' script returned exit code 143 public/default ++ date + echo '[Sat May 4 21:18:53 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:18:53 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.3359.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-owner-resign-test-14651?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 table cdc_tiflash_test.multi_data_type exists check diff failed 1-th time, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes check diff failed 2-th time, retry later check diff successfully {"level":"warn","ts":1714828733.3529289,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020d1dc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714828733.6230385,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002317880/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 Create changefeed successfully! ID: 70f31876-c369-4118-8143-a219e3e2fbaf Info: {"upstream_id":7365133211649941112,"namespace":"default","id":"70f31876-c369-4118-8143-a219e3e2fbaf","sink_uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-14651?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:18:53.980706973+08:00","start_ts":449532063484739585,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532063484739585,"checkpoint_ts":449532063484739585,"checkpoint_time":"2024-05-04 21:18:53.386"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 13:18:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {UpstreamID:7365133193938033584 Namespace:default ID:9a4d152b-6fdc-4456-b817-c0b5c3a1e821 SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-19193?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:18:46.873350592 +0800 CST StartTs:449532061619060738 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004604480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532061776347138} {CheckpointTs:449532063676629006 MinTableBarrierTs:449532063676629006 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/314daaf0-8644-41ce-9f3b-5d3349fb1649 {"id":"314daaf0-8644-41ce-9f3b-5d3349fb1649","address":"127.0.0.1:8302","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828731} /tidb/cdc/default/__cdc_meta__/capture/a5187ff8-6e37-406b-ba89-2a257576c18e {"id":"a5187ff8-6e37-406b-ba89-2a257576c18e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c1710ccd a5187ff8-6e37-406b-ba89-2a257576c18e /tidb/cdc/default/__cdc_meta__/owner/22318f43c1710d79 314daaf0-8644-41ce-9f3b-5d3349fb1649 /tidb/cdc/default/default/changefeed/info/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"upstream-id":7365133193938033584,"namespace":"default","changefeed-id":"9a4d152b-6fdc-4456-b817-c0b5c3a1e821","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-19193?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:18:46.873350592+08:00","start-ts":449532061619060738,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532061776347138} /tidb/cdc/default/default/changefeed/status/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"checkpoint-ts":449532063676629006,"min-table-barrier-ts":449532063676629006,"admin-job-type":0} /tidb/cdc/default/default/task/position/314daaf0-8644-41ce-9f3b-5d3349fb1649/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a5187ff8-6e37-406b-ba89-2a257576c18e/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365133193938033584 {"id":7365133193938033584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {UpstreamID:7365133193938033584 Namespace:default ID:9a4d152b-6fdc-4456-b817-c0b5c3a1e821 SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-19193?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:18:46.873350592 +0800 CST StartTs:449532061619060738 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004604480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532061776347138} {CheckpointTs:449532063676629006 MinTableBarrierTs:449532063676629006 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/314daaf0-8644-41ce-9f3b-5d3349fb1649 {"id":"314daaf0-8644-41ce-9f3b-5d3349fb1649","address":"127.0.0.1:8302","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828731} /tidb/cdc/default/__cdc_meta__/capture/a5187ff8-6e37-406b-ba89-2a257576c18e {"id":"a5187ff8-6e37-406b-ba89-2a257576c18e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c1710ccd a5187ff8-6e37-406b-ba89-2a257576c18e /tidb/cdc/default/__cdc_meta__/owner/22318f43c1710d79 314daaf0-8644-41ce-9f3b-5d3349fb1649 /tidb/cdc/default/default/changefeed/info/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"upstream-id":7365133193938033584,"namespace":"default","changefeed-id":"9a4d152b-6fdc-4456-b817-c0b5c3a1e821","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-19193?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:18:46.873350592+08:00","start-ts":449532061619060738,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532061776347138} /tidb/cdc/default/default/changefeed/status/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"checkpoint-ts":449532063676629006,"min-table-barrier-ts":449532063676629006,"admin-job-type":0} /tidb/cdc/default/default/task/position/314daaf0-8644-41ce-9f3b-5d3349fb1649/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a5187ff8-6e37-406b-ba89-2a257576c18e/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365133193938033584 {"id":7365133193938033584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {UpstreamID:7365133193938033584 Namespace:default ID:9a4d152b-6fdc-4456-b817-c0b5c3a1e821 SinkURI:pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-19193?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 21:18:46.873350592 +0800 CST StartTs:449532061619060738 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004604480 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-91-g7f3a787ed Epoch:449532061776347138} {CheckpointTs:449532063676629006 MinTableBarrierTs:449532063676629006 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/314daaf0-8644-41ce-9f3b-5d3349fb1649 {"id":"314daaf0-8644-41ce-9f3b-5d3349fb1649","address":"127.0.0.1:8302","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828731} /tidb/cdc/default/__cdc_meta__/capture/a5187ff8-6e37-406b-ba89-2a257576c18e {"id":"a5187ff8-6e37-406b-ba89-2a257576c18e","address":"127.0.0.1:8301","version":"v8.2.0-alpha-91-g7f3a787ed","git-hash":"7f3a787edb079ab2f7719b4b79f735e1697cc211","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_pulsar_test/tiflow/bin/cdc.test","start-timestamp":1714828716} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f43c1710ccd a5187ff8-6e37-406b-ba89-2a257576c18e /tidb/cdc/default/__cdc_meta__/owner/22318f43c1710d79 314daaf0-8644-41ce-9f3b-5d3349fb1649 /tidb/cdc/default/default/changefeed/info/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"upstream-id":7365133193938033584,"namespace":"default","changefeed-id":"9a4d152b-6fdc-4456-b817-c0b5c3a1e821","sink-uri":"pulsar://127.0.0.1:6650/ticdc-processor-resolved-ts-fallback-test-19193?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T21:18:46.873350592+08:00","start-ts":449532061619060738,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-91-g7f3a787ed","epoch":449532061776347138} /tidb/cdc/default/default/changefeed/status/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"checkpoint-ts":449532063676629006,"min-table-barrier-ts":449532063676629006,"admin-job-type":0} /tidb/cdc/default/default/task/position/314daaf0-8644-41ce-9f3b-5d3349fb1649/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/a5187ff8-6e37-406b-ba89-2a257576c18e/9a4d152b-6fdc-4456-b817-c0b5c3a1e821 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365133193938033584 {"id":7365133193938033584,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x public/default check diff successfully ++ date + echo '[Sat May 4 21:18:55 CST 2024] <<<<<< pulsar is ready >>>>>>' [Sat May 4 21:18:55 CST 2024] <<<<<< pulsar is ready >>>>>> + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.3413.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-26154?protocol=canal-json&enable-tidb-extension=true' + set +x [Sat May 4 21:18:55 CST 2024] <<<<<< START Pulsar consumer in owner_resign case >>>>>> table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes Create changefeed successfully! ID: 507db434-1606-472d-ad97-d12b936b9b01 Info: {"upstream_id":7365133226535938941,"namespace":"default","id":"507db434-1606-472d-ad97-d12b936b9b01","sink_uri":"pulsar://127.0.0.1:6650/ticdc-batch-add-table-test-26154?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T21:18:56.221810966+08:00","start_ts":449532064070893570,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-91-g7f3a787ed","resolved_ts":449532064070893570,"checkpoint_ts":449532064070893570,"checkpoint_time":"2024-05-04 21:18:55.622"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... {"level":"warn","ts":1714828736.3915942,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002ab9c00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714828736.6614385,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00237efc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714828737.2079864,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021a6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 {"level":"warn","ts":1714828737.4823148,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022d6000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G04' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G05' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G06' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G12' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G13' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G14' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G15' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G16' [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G17' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline ERROR: script returned exit code 1 Finished: FAILURE