Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_storage_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-npfcd-chv4p --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ec4451d3d3df559276fb7bde5af7f8b14a176012" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-npfcd" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=272b3817-2d4c-49ad-bfbb-57bb440e9d35 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-npfcd GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-ec41a9ea2975eb575c6c655de75271b1a69674254216af41486e1697505d3c9f NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-npfcd pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build454778182=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-storage-test-1781-npf-20xzc bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 62769 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 846debc4a93dcdc0 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Sat May 04 08:35:00 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/model/kv.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sourcemanager/manager.go, cdc/redo/reader/reader.go, cdc/sink/dmlsink/factory/factory.go, cdc/sink/dmlsink/txn/mysql/dml.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/kafka-consumer/main.go, cmd/pulsar-consumer/main.go, cmd/storage-consumer/main.go, errors.toml, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/errors/cdc_errors.go, pkg/errors/helper.go, tests/integration_tests/_utils/check_sync_diff, tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml, tests/integration_tests/changefeed_dup_error_restart/conf/workload, tests/integration_tests/changefeed_dup_error_restart/run.sh, tests/integration_tests/force_replicate_table/run.sh, tests/integration_tests/open_protocol_handle_key_only/data/data.sql, tests/integration_tests/open_protocol_handle_key_only/run.sh, tests/integration_tests/run_group.sh] [Pipeline] echo diff file not matched: cdc/model/kv.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534) 203635712 bytes in 1.19 secs (171162305 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/.git/ .git HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) POST git-upload-pack (656 bytes) POST git-upload-pack (973 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master * [new ref] refs/pull/10919/head -> origin/pr/10919/head HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) ๐Ÿšง Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) ๐Ÿšง Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..c950cce3a Fast-forward cdc/model/kv.go | 5 + cdc/model/sink.go | 38 ++- cdc/model/sink_test.go | 9 +- cdc/processor/processor.go | 21 +- cdc/processor/sinkmanager/manager.go | 5 + cdc/processor/sourcemanager/manager.go | 66 +++- cdc/redo/reader/reader.go | 21 +- cdc/sink/dmlsink/factory/factory.go | 8 +- cdc/sink/dmlsink/txn/mysql/dml.go | 7 + cdc/sink/dmlsink/txn/mysql/mysql.go | 87 +++--- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 2 +- cmd/kafka-consumer/main.go | 4 +- cmd/pulsar-consumer/main.go | 17 +- cmd/storage-consumer/main.go | 4 +- errors.toml | 5 + pkg/applier/redo.go | 303 +++++++++++++++++- pkg/applier/redo_test.go | 347 ++++++++++++++++++++- pkg/errors/cdc_errors.go | 4 + pkg/errors/helper.go | 19 ++ tests/integration_tests/_utils/check_sync_diff | 2 +- .../conf/diff_config.toml | 29 ++ .../changefeed_dup_error_restart/conf/workload | 13 + .../changefeed_dup_error_restart/run.sh | 54 ++++ .../integration_tests/force_replicate_table/run.sh | 4 +- .../open_protocol_handle_key_only/data/data.sql | 2 +- .../open_protocol_handle_key_only/run.sh | 2 + tests/integration_tests/run_group.sh | 5 +- 27 files changed, 980 insertions(+), 103 deletions(-) create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/workload create mode 100755 tests/integration_tests/changefeed_dup_error_restart/run.sh ๐Ÿงพ Pre-merged result: c950cce3a9b105fd95bb2c788e1ab69ec32e0668 c950cce3a f 4d7b2cab4 f b673ef40a fix โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache saved successfully (git/pingcap/tiflow/rev-be15534-c950cce) 203849728 bytes in 8.31 secs (24541548 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1301 0 --:--:-- --:--:-- --:--:-- 1322 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1894 0 --:--:-- --:--:-- --:--:-- 1952 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2083 0 --:--:-- --:--:-- --:--:-- 2157 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 2361 0 --:--:-- --:--:-- --:--:-- 2277 100 41 100 41 0 0 2350 0 --:--:-- --:--:-- --:--:-- 2277 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz 2024-05-04 16:35:25 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz [536570515/536570515] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-04 16:35:36 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz 2024-05-04 16:35:49 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz [919098782/919098782] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz 2024-05-04 16:36:06 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz [456057803/456057803] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-04 16:36:11 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-04 16:36:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-04 16:36:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-04 16:36:12 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-04 16:36:14 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-04 16:36:21 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 16:36 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 16:36 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 16:36 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 16:36 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 16:36 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 16:36 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 16:36 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 16:36 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 8.0K drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 16:36 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 16:36 .. + '[' -f ./bin/cdc ']' + make cdc CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-79-gc950cce3a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 08:36:24" -X "github.com/pingcap/tiflow/pkg/version.GitHash=c950cce3a9b105fd95bb2c788e1ab69ec32e0668" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-79-gc950cce3a"' -o bin/cdc ./cmd/cdc go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading golang.org/x/net v0.24.0 go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading golang.org/x/sync v0.7.0 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/DATA-DOG/go-sqlmock v1.5.0 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/imdario/mergo v0.3.16 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/google/btree v1.1.2 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/r3labs/diff v1.1.0 go: downloading github.com/swaggo/files v0.0.0-20210815190702-a29dd2bc99b2 go: downloading github.com/swaggo/gin-swagger v1.2.0 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/hashicorp/golang-lru v0.5.1 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading github.com/YangKeao/seahash v0.0.0-20240229041150-e7bf269c3140 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/segmentio/kafka-go v0.4.41-0.20230526171612-f057b1d369cd go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/goccy/go-json v0.10.2 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading golang.org/x/sys v0.19.0 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/pingcap/check v0.0.0-20211026125417-57bd13f7b5f0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading github.com/uber-go/atomic v1.4.0 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/golang/mock v1.6.0 go: downloading github.com/swaggo/swag v1.16.3 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/pingcap/tidb-dashboard v0.0.0-20240326110213-9768844ff5d7 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading google.golang.org/protobuf v1.33.0 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/aws/smithy-go v1.13.5 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading google.golang.org/api v0.170.0 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/xdg-go/scram v1.1.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading golang.org/x/text v0.14.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda go: downloading github.com/aws/aws-sdk-go-v2/config v1.18.30 go: downloading github.com/aws/aws-sdk-go-v2/credentials v1.13.29 go: downloading github.com/aws/aws-sdk-go-v2/service/glue v1.58.1 go: downloading github.com/jarcoal/httpmock v1.2.0 go: downloading github.com/mailru/easyjson v0.7.7 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/KyleBanks/depth v1.2.1 go: downloading github.com/go-openapi/spec v0.21.0 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/joomcode/errorx v1.0.1 go: downloading github.com/glebarez/sqlite v1.7.0 go: downloading gorm.io/driver/mysql v1.3.3 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/99designs/keyring v1.2.1 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/xdg-go/pbkdf2 v1.0.0 go: downloading github.com/xdg-go/stringprep v1.0.4 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/shopspring/decimal v1.3.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/blacktear23/go-proxyprotocol v1.0.6 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/pingcap/fn v1.0.0 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading github.com/glebarez/go-sqlite v1.21.2 go: downloading github.com/go-ozzo/ozzo-validation/v4 v4.3.0 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 go: downloading github.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/ardielle/ardielle-go v1.5.2 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/go-playground/locales v0.14.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.6 go: downloading github.com/aws/aws-sdk-go-v2/internal/ini v1.3.37 go: downloading github.com/aws/aws-sdk-go-v2/service/sso v1.12.14 go: downloading github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.14 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/aws/aws-sdk-go-v2/service/sts v1.20.1 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/go-openapi/jsonpointer v0.21.0 go: downloading github.com/go-openapi/jsonreference v0.21.0 go: downloading github.com/go-openapi/swag v0.23.0 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.36 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading modernc.org/libc v1.37.1 go: downloading modernc.org/sqlite v1.27.0 go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.30 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.30 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/josharian/intern v1.0.0 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading github.com/google/gofuzz v1.2.0 go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading modernc.org/memory v1.7.2 go: downloading modernc.org/mathutil v1.6.0 go: downloading github.com/jmespath/go-jmespath v0.4.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 + '[' -f ./bin/cdc_kafka_consumer ']' + make kafka_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-79-gc950cce3a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 08:39:29" -X "github.com/pingcap/tiflow/pkg/version.GitHash=c950cce3a9b105fd95bb2c788e1ab69ec32e0668" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-79-gc950cce3a"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go + '[' -f ./bin/cdc_storage_consumer ']' + make storage_consumer CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-79-gc950cce3a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 08:39:37" -X "github.com/pingcap/tiflow/pkg/version.GitHash=c950cce3a9b105fd95bb2c788e1ab69ec32e0668" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-79-gc950cce3a"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go + '[' -f ./bin/cdc.test ']' + make integration_test_build cd tools/check && GO111MODULE=on go build -mod=mod -o ../bin/failpoint-ctl github.com/pingcap/failpoint/failpoint-ctl go: downloading github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd go: downloading github.com/sergi/go-diff v1.1.0 CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-79-gc950cce3a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 08:39:45" -X "github.com/pingcap/tiflow/pkg/version.GitHash=c950cce3a9b105fd95bb2c788e1ab69ec32e0668" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-79-gc950cce3a"' -o bin/cdc_storage_consumer ./cmd/storage-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-79-gc950cce3a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 08:39:45" -X "github.com/pingcap/tiflow/pkg/version.GitHash=c950cce3a9b105fd95bb2c788e1ab69ec32e0668" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-79-gc950cce3a"' -o bin/cdc_kafka_consumer ./cmd/kafka-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-79-gc950cce3a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 08:39:45" -X "github.com/pingcap/tiflow/pkg/version.GitHash=c950cce3a9b105fd95bb2c788e1ab69ec32e0668" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-79-gc950cce3a"' -o bin/cdc_pulsar_consumer ./cmd/pulsar-consumer/main.go CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-79-gc950cce3a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 08:39:45" -X "github.com/pingcap/tiflow/pkg/version.GitHash=c950cce3a9b105fd95bb2c788e1ab69ec32e0668" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-79-gc950cce3a"' -o bin/oauth2-server ./cmd/oauth2-server/main.go go: downloading github.com/go-oauth2/oauth2/v4 v4.5.2 go: downloading github.com/tidwall/buntdb v1.3.0 go: downloading github.com/tidwall/rtred v0.1.2 go: downloading github.com/tidwall/grect v0.1.4 go: downloading github.com/tidwall/gjson v1.14.3 go: downloading github.com/tidwall/match v1.1.1 go: downloading github.com/tidwall/tinyqueue v0.1.1 go: downloading github.com/tidwall/pretty v1.2.0 $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl enable >/dev/null) go: downloading github.com/PingCAP-QE/go-sqlsmith v0.0.0-20231213065948-336e064b488d go: downloading github.com/chzyer/readline v1.5.1 go: downloading github.com/deepmap/oapi-codegen v1.9.0 go: downloading github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 go: downloading github.com/getkin/kin-openapi v0.80.0 go: downloading github.com/gogo/gateway v1.1.0 go: downloading github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 go: downloading go.uber.org/dig v1.13.0 go: downloading github.com/mattn/go-shellwords v1.0.12 go: downloading go.uber.org/ratelimit v0.2.0 go: downloading github.com/VividCortex/mysqlerr v1.0.0 go: downloading github.com/DataDog/zstd v1.5.5 go: downloading go.uber.org/goleak v1.3.0 go: downloading github.com/shurcooL/httpgzip v0.0.0-20190720172056-320755c1c1b0 go: downloading github.com/bradleyjkemp/grpc-tools v0.2.5 go: downloading github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 go: downloading github.com/integralist/go-findroot v0.0.0-20160518114804-ac90681525dc go: downloading github.com/jmoiron/sqlx v1.3.3 go: downloading upper.io/db.v3 v3.7.1+incompatible go: downloading github.com/ngaut/log v0.0.0-20210830112240-0124ec040aeb go: downloading github.com/improbable-eng/grpc-web v0.12.0 go: downloading github.com/ghodss/yaml v1.0.0 go: downloading github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f go: downloading github.com/rs/cors v1.7.0 CGO_ENABLED=1 GO111MODULE=on go test -p 3 --race --tags=intest -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-79-gc950cce3a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 08:39:45" -X "github.com/pingcap/tiflow/pkg/version.GitHash=c950cce3a9b105fd95bb2c788e1ab69ec32e0668" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-79-gc950cce3a"' -c -cover -covermode=atomic \ -coverpkg=github.com/pingcap/tiflow/... \ -o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } CGO_ENABLED=0 GO111MODULE=on go build -trimpath -ldflags '-X "github.com/pingcap/tiflow/pkg/version.ReleaseVersion=v8.2.0-alpha-79-gc950cce3a" -X "github.com/pingcap/tiflow/pkg/version.BuildTS=2024-05-04 08:39:45" -X "github.com/pingcap/tiflow/pkg/version.GitHash=c950cce3a9b105fd95bb2c788e1ab69ec32e0668" -X "github.com/pingcap/tiflow/pkg/version.GitBranch=HEAD" -X "github.com/pingcap/tiflow/pkg/version.GoVersion=go version go1.21.6 linux/amd64" -X "github.com/pingcap/tidb/pkg/parser/mysql.TiDBReleaseVersion=v8.2.0-alpha-79-gc950cce3a"' -o bin/cdc ./cmd/cdc/main.go \ || { $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null); exit 1; } $(echo $(for p in $(go list ./... | grep -vE 'vendor|proto|tiflow/tests|integration|testing_utils|pb|pbmock|tiflow/bin'); do echo ${p#"github.com/pingcap/tiflow/"}|grep -v "github.com/pingcap/tiflow"; done) | xargs tools/bin/failpoint-ctl disable >/dev/null) + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 16:45 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 16:36 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 4 16:45 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 16:44 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 16:40 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 16:40 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 16:39 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 4 16:40 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-79-gc950cce3a Git Commit Hash: c950cce3a9b105fd95bb2c788e1ab69ec32e0668 Git Branch: HEAD UTC Build Time: 2024-05-04 08:39:45 Go Version: go version go1.21.6 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-be15534-c950cce already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 4 16:45 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 4 16:36 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 4 16:45 bin -rwxr-xr-x. 1 jenkins jenkins 220M May 4 16:45 cdc -rwxr-xr-x. 1 jenkins jenkins 363M May 4 16:44 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 16:40 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 16:40 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 16:39 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 16:45 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 4 16:45 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 4 16:45 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 4 16:45 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 4 16:45 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 4 16:45 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 4 16:45 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 4 16:45 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 4 16:45 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 4 16:45 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 4 16:45 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 4 16:45 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 4 16:45 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 4 16:45 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 4 16:40 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 4 16:45 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 4 16:45 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 4 16:45 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 4 16:45 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 4 16:45 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 4 16:45 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M May 4 16:45 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 4 16:45 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 4 16:45 share -rwxr-xr-x. 1 jenkins jenkins 32M May 4 16:45 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M May 4 16:45 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 4 16:45 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 4 16:45 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 4 16:45 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 4 16:45 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 72.86 secs (51202186 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-jwt-lwq1s [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7 [Pipeline] { [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-lz6hc-l3cmt --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6344a69120dc298c7cebc6c9a01869914ac305ae" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-lz6hc" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3926e824; decorates RemoteLauncher[hudson.remoting.Channel@5c8862a4:JNLP4-connect connection from 10.233.123.4/10.233.123.4:39596] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] node Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-jwt-lwq1s is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-jwt27-54nw4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "e8d2b36103b458eff481fa37cf50263bd19cb724" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-jwt27" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-jwt-lwq1s" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-jwt-lwq1s" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-jwt-lwq1s" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-jwt-lwq1s in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs [Pipeline] podTemplate [Pipeline] { [Pipeline] node No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7062ca60; decorates RemoteLauncher[hudson.remoting.Channel@7d146709:JNLP4-connect connection from 10.233.66.24/10.233.66.24:38504] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] podTemplate [Pipeline] { [Pipeline] node [Pipeline] withEnv [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-lsdkl-4xglh --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "19d39dba1fd6e757066771a82a16e9bf5e8526ad" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-lsdkl" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-bpq0z-wrvhp --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "99e355f27854569be8c48c9994e2c77d57e12625" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-bpq0z" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-w70sv-17nvj --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "04d740bfe7fd45674fa8d783ab0d2a8b743876f7" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-w70sv" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-pqsx2-4hkr6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "59b2c37a935c161b97749b2b3d1f87f930ada7ff" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-pqsx2" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-7svrs-0q4b7 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "c1221278fdc5fd2731a98f2e5c731ee562bcdb23" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-7svrs" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-qv0bl-xlwm6 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d0edc9c0cd9799d3e2ae2fa8a2eb744a05f406e7" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-qv0bl" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-80lm2-2x9d4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6d61a03bec07a2460ac80a06e2a1a6b4406ad5e5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-80lm2" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-jjg7q-3c8x8 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f2d2f45890a26137324e551aeeb292d1e65a97ef" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-jjg7q" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 21.81 secs (171063822 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=f647ab06-dcc5-4d15-8587-1c0496583c6c BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-lz6hc GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-lz6hc pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:46:59 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] checkout [Pipeline] { The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@af72461; decorates RemoteLauncher[hudson.remoting.Channel@259ce2bb:JNLP4-connect connection from 10.233.100.31/10.233.100.31:33748] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@32c71a15; decorates RemoteLauncher[hudson.remoting.Channel@14809dc4:JNLP4-connect connection from 10.233.97.120/10.233.97.120:49488] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@46bc65a3; decorates RemoteLauncher[hudson.remoting.Channel@30aa4d26:JNLP4-connect connection from 10.233.127.174/10.233.127.174:49106] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@770417c3; decorates RemoteLauncher[hudson.remoting.Channel@604f681f:JNLP4-connect connection from 10.233.93.207/10.233.93.207:40854] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@766546d2; decorates RemoteLauncher[hudson.remoting.Channel@3cedf887:JNLP4-connect connection from 10.233.107.72/10.233.107.72:60558] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@2ed73304; decorates RemoteLauncher[hudson.remoting.Channel@7055eccd:JNLP4-connect connection from 10.233.105.112/10.233.105.112:51422] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3d13c83f; decorates RemoteLauncher[hudson.remoting.Channel@62463b9c:JNLP4-connect connection from 10.233.90.157/10.233.90.157:52438] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6877b81a; decorates RemoteLauncher[hudson.remoting.Channel@32e00e95:JNLP4-connect connection from 10.233.71.83/10.233.71.83:59790] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: storage... <<================= > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 [Sat May 4 16:47:03 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-n0x62-0fjnl --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "67e30345fe4814b795277693ac0e8ad538ca5f95" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-n0x62" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-5v88p-36m6q --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6c762b086428190dbf7bb4520e9792a3f6aabae7" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-5v88p" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-pb1r6-1kzr2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6fbeac86e91c6f6b36dcddad8fea109ebaa62d83" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-pb1r6" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-rlkxk-lpmcb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0059073eaae3a9003bcceceb8dc67fbf2484e3ca" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-rlkxk" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: storage... <<================= kafka downstream isn't support syncpoint record [Sat May 4 16:47:06 CST 2024] <<<<<< run test case syncpoint success! >>>>>> > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-bzq0f-9frfc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7fa377a488bea37c40b9edbbab05aa5124382806" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-bzq0f" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:09 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-kh4bj-0m71k --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7076717729f6514f6f799a33c930a489ec35acc5" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-kh4bj" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 7.20 secs (518357640 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G01 Run cases: open_protocol_handle_key_only PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=a520e027-467a-4c6f-a641-903a1fd68cc6 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-jwt-lwq1s HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-jwt27 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1781-jwt-lwq1s pingcap_tiflow_pull_cdc_integration_storage_test_1781-jwt27 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-jwt-lwq1s GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:11 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-nbjsn-38t2v --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "6f638466b38e1956ffdfc113cb3649915dc303da" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-nbjsn" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] withEnv [Pipeline] { Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7febf85b; decorates RemoteLauncher[hudson.remoting.Channel@1e3a42e3:JNLP4-connect connection from 10.233.90.31/10.233.90.31:50282] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] checkout The recommended git tool is: git [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3bac7e92; decorates RemoteLauncher[hudson.remoting.Channel@66fdc118:JNLP4-connect connection from 10.233.88.92/10.233.88.92:47072] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials Agent pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1781-29bjp-2pcmm --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "f5e855f9f627cb1fd75a1702fec60854d9cd0b79" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1781-29bjp" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4026356; decorates RemoteLauncher[hudson.remoting.Channel@4c41cb8a:JNLP4-connect connection from 10.233.106.96/10.233.106.96:40374] will be ignored (a typical symptom is the Git executable not being run inside a designated container) [Pipeline] withCredentials Running on pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3a1a440d; decorates RemoteLauncher[hudson.remoting.Channel@70a748cf:JNLP4-connect connection from 10.233.100.233/10.233.100.233:41404] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@b8b7ca1; decorates RemoteLauncher[hudson.remoting.Channel@71ab7cf6:JNLP4-connect connection from 10.233.127.137/10.233.127.137:56190] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@46aedd70; decorates RemoteLauncher[hudson.remoting.Channel@bf64d5d:JNLP4-connect connection from 10.233.105.120/10.233.105.120:55322] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:12 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { [Pipeline] { [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] checkout [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { The recommended git tool is: git [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@36922069; decorates RemoteLauncher[hudson.remoting.Channel@30e50102:JNLP4-connect connection from 10.233.71.118/10.233.71.118:38950] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6de94c32; decorates RemoteLauncher[hudson.remoting.Channel@292e4ba9:JNLP4-connect connection from 10.233.67.113/10.233.67.113:57964] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) \033[0;36m<<< Run all test success >>>\033[0m Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:15 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:18 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:22 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:25 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:28 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:32 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:35 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 23.67 secs (157581389 bytes/sec) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:39 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:47:42 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: storage... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/lossy_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b278fa40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z, pid:2480, start at 2024-05-04 16:47:58.468138661 +0800 CST m=+5.289113713 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:49:58.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:47:58.441 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:37:58.441 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 19.40 secs (192321827 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b278fa40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z, pid:2480, start at 2024-05-04 16:47:58.468138661 +0800 CST m=+5.289113713 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:49:58.477 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:47:58.441 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:37:58.441 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2791280014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z, pid:2562, start at 2024-05-04 16:47:58.564664468 +0800 CST m=+5.312310752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:49:58.574 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:47:58.538 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:37:58.538 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/lossy_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/lossy_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/lossy_ddl/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/lossy_ddl/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/lossy_ddl/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sat May 4 16:48:03 CST 2024] <<<<<< START cdc server in lossy_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.lossy_ddl.38053807.out server --log-file /tmp/tidb_cdc_test/lossy_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/lossy_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:48:06 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6856faee-29d6-47bb-8eb5-93fa60595614 {"id":"6856faee-29d6-47bb-8eb5-93fa60595614","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42c9bf5cce 6856faee-29d6-47bb-8eb5-93fa60595614 /tidb/cdc/default/default/upstream/7365063480453085758 {"id":7365063480453085758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6856faee-29d6-47bb-8eb5-93fa60595614 {"id":"6856faee-29d6-47bb-8eb5-93fa60595614","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42c9bf5cce 6856faee-29d6-47bb-8eb5-93fa60595614 /tidb/cdc/default/default/upstream/7365063480453085758 {"id":7365063480453085758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6856faee-29d6-47bb-8eb5-93fa60595614 {"id":"6856faee-29d6-47bb-8eb5-93fa60595614","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812483} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42c9bf5cce 6856faee-29d6-47bb-8eb5-93fa60595614 /tidb/cdc/default/default/upstream/7365063480453085758 {"id":7365063480453085758,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: a3d9e010-7d53-4012-9198-4d07bf746d7e Info: {"upstream_id":7365063480453085758,"namespace":"default","id":"a3d9e010-7d53-4012-9198-4d07bf746d7e","sink_uri":"blackhole:","create_time":"2024-05-04T16:48:06.805018912+08:00","start_ts":449527804510863365,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527804510863365,"checkpoint_ts":449527804510863365,"checkpoint_time":"2024-05-04 16:48:06.690"} Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 8.01 secs (465768156 bytes/sec) [Pipeline] { [Pipeline] cache check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 1-th time, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 2-th time, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 3-th time, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 4-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 24.65 secs (151335977 bytes/sec) [Pipeline] { [Pipeline] cache check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 5-th time, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 6-th time, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl [2024/05/04 16:48:10.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527804589768718,\"CommitTs\":449527804589768719,\"RowID\":1,\"PhysicalTableID\":106,\"TableInfo\":{\"id\":106,\"name\":{\"O\":\"example1\",\"L\":\"example1\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527804563292197,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example1\",\"TableID\":106,\"IsPartition\":false},\"Version\":449527804576399367,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449527805481058335,\"HandleKey\":1}"] [2024/05/04 16:48:13.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527804681519123,\"CommitTs\":449527804681519125,\"RowID\":1,\"PhysicalTableID\":109,\"TableInfo\":{\"id\":109,\"name\":{\"O\":\"example2\",\"L\":\"example2\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527804668149789,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example2\",\"TableID\":109,\"IsPartition\":false},\"Version\":449527804668149793,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449527806267490309,\"HandleKey\":1}"] [2024/05/04 16:48:15.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527804786114572,\"CommitTs\":449527804786114574,\"RowID\":1,\"PhysicalTableID\":112,\"TableInfo\":{\"id\":112,\"name\":{\"O\":\"example3\",\"L\":\"example3\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":7,\"Flag\":128,\"Flen\":19,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527804760162338,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example3\",\"TableID\":112,\"IsPartition\":false},\"Version\":449527804760162342,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"2023-04-19 11:48:00\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":36,\"SplitTxn\":false,\"ReplicatingTs\":449527806791778309,\"HandleKey\":1}"] [2024/05/04 16:48:17.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527804878127121,\"CommitTs\":449527804878127123,\"RowID\":1,\"PhysicalTableID\":115,\"TableInfo\":{\"id\":115,\"name\":{\"O\":\"example4\",\"L\":\"example4\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527804865019926,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example4\",\"TableID\":115,\"IsPartition\":false},\"Version\":449527804865019931,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449527807316066313,\"HandleKey\":1}"] [2024/05/04 16:48:19.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527804982722566,\"CommitTs\":449527804982722568,\"RowID\":1,\"PhysicalTableID\":118,\"TableInfo\":{\"id\":118,\"name\":{\"O\":\"example5\",\"L\":\"example5\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527804956770345,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example5\",\"TableID\":118,\"IsPartition\":false},\"Version\":449527804956770349,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449527807840354320,\"HandleKey\":1}"] [2024/05/04 16:48:21.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805061365777,\"CommitTs\":449527805061365779,\"RowID\":1,\"PhysicalTableID\":121,\"TableInfo\":{\"id\":121,\"name\":{\"O\":\"example6\",\"L\":\"example6\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805048520735,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example6\",\"TableID\":121,\"IsPartition\":false},\"Version\":449527805048520739,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449527808364642309,\"HandleKey\":1}"] [2024/05/04 16:48:23.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805153378319,\"CommitTs\":449527805153378321,\"RowID\":1,\"PhysicalTableID\":124,\"TableInfo\":{\"id\":124,\"name\":{\"O\":\"example7\",\"L\":\"example7\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8\",\"Collate\":\"utf8_general_ci\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805140008979,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example7\",\"TableID\":124,\"IsPartition\":false},\"Version\":449527805140008984,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449527808888930313,\"HandleKey\":1}"] [2024/05/04 16:48:25.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805192699921,\"CommitTs\":449527805192699923,\"RowID\":-1,\"PhysicalTableID\":128,\"TableInfo\":{\"id\":127,\"name\":{\"O\":\"example8\",\"L\":\"example8\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805179592730,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":128,\"name\":{\"O\":\"b0\",\"L\":\"b0\"},\"less_than\":[\"0\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":129,\"name\":{\"O\":\"b1\",\"L\":\"b1\"},\"less_than\":[\"MAXVALUE\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":2,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example8\",\"TableID\":127,\"IsPartition\":true},\"Version\":449527805179592734,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":-1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449527809413218309,\"HandleKey\":-1}"] [2024/05/04 16:48:25.524 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805192699924,\"CommitTs\":449527805192699925,\"RowID\":1,\"PhysicalTableID\":129,\"TableInfo\":{\"id\":127,\"name\":{\"O\":\"example8\",\"L\":\"example8\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805179592730,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":128,\"name\":{\"O\":\"b0\",\"L\":\"b0\"},\"less_than\":[\"0\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":129,\"name\":{\"O\":\"b1\",\"L\":\"b1\"},\"less_than\":[\"MAXVALUE\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":2,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example8\",\"TableID\":127,\"IsPartition\":true},\"Version\":449527805179592734,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449527809413218310,\"HandleKey\":1}"] [2024/05/04 16:48:27.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805271080974,\"CommitTs\":449527805271080975,\"RowID\":1,\"PhysicalTableID\":132,\"TableInfo\":{\"id\":132,\"name\":{\"O\":\"example9\",\"L\":\"example9\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805244866599,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example9\",\"TableID\":132,\"IsPartition\":false},\"Version\":449527805244866603,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449527809937506309,\"HandleKey\":1}"] [2024/05/04 16:48:29.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805310664718,\"CommitTs\":449527805310664719,\"RowID\":1,\"PhysicalTableID\":135,\"TableInfo\":{\"id\":135,\"name\":{\"O\":\"example10\",\"L\":\"example10\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":5,\"Flag\":0,\"Flen\":22,\"Decimal\":-1,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805284188193,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example10\",\"TableID\":135,\"IsPartition\":false},\"Version\":449527805297557505,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":36,\"SplitTxn\":false,\"ReplicatingTs\":449527810461794314,\"HandleKey\":1}"] [2024/05/04 16:48:31.524 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805402415120,\"CommitTs\":449527805402415122,\"RowID\":1,\"PhysicalTableID\":138,\"TableInfo\":{\"id\":138,\"name\":{\"O\":\"example11\",\"L\":\"example11\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":8,\"Flag\":4099,\"Flen\":20,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":8,\"Flag\":0,\"Flen\":20,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805389045783,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example11\",\"TableID\":138,\"IsPartition\":false},\"Version\":449527805389045787,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449527810986082309,\"HandleKey\":1}"] [2024/05/04 16:48:33.524 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805507272717,\"CommitTs\":449527805507272719,\"RowID\":1,\"PhysicalTableID\":141,\"TableInfo\":{\"id\":141,\"name\":{\"O\":\"example12\",\"L\":\"example12\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":251,\"Flag\":0,\"Flen\":4294967295,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805481058338,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example12\",\"TableID\":141,\"IsPartition\":false},\"Version\":449527805481058343,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449527811510370309,\"HandleKey\":1}"] [2024/05/04 16:48:35.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805599023119,\"CommitTs\":449527805599023121,\"RowID\":1,\"PhysicalTableID\":144,\"TableInfo\":{\"id\":144,\"name\":{\"O\":\"example13\",\"L\":\"example13\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":247,\"Flag\":0,\"Flen\":1,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":[\"a\",\"b\",\"c\"],\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805585653779,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example13\",\"TableID\":144,\"IsPartition\":false},\"Version\":449527805585653786,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449527812047765513,\"HandleKey\":1}"] [2024/05/04 16:48:37.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805690773521,\"CommitTs\":449527805690773523,\"RowID\":1,\"PhysicalTableID\":147,\"TableInfo\":{\"id\":147,\"name\":{\"O\":\"example14\",\"L\":\"example14\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805677404193,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example14\",\"TableID\":147,\"IsPartition\":false},\"Version\":449527805677404198,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449527812572053519,\"HandleKey\":1}"] [2024/05/04 16:48:39.524 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805795631118,\"CommitTs\":449527805795631120,\"RowID\":1,\"PhysicalTableID\":150,\"TableInfo\":{\"id\":150,\"name\":{\"O\":\"example15\",\"L\":\"example15\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805782523909,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example15\",\"TableID\":150,\"IsPartition\":false},\"Version\":449527805782523913,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449527813096341513,\"HandleKey\":1}"] [2024/05/04 16:48:41.524 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805887381519,\"CommitTs\":449527805887381521,\"RowID\":1,\"PhysicalTableID\":154,\"TableInfo\":{\"id\":153,\"name\":{\"O\":\"example16\",\"L\":\"example16\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805874012182,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`a`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":154,\"name\":{\"O\":\"p0\",\"L\":\"p0\"},\"less_than\":[\"6\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":155,\"name\":{\"O\":\"p1\",\"L\":\"p1\"},\"less_than\":[\"11\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":156,\"name\":{\"O\":\"p2\",\"L\":\"p2\"},\"less_than\":[\"21\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":3,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example16\",\"TableID\":153,\"IsPartition\":true},\"Version\":449527805874012186,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449527813620367370,\"HandleKey\":1}"] [2024/05/04 16:48:43.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805913595912,\"CommitTs\":449527805913595913,\"RowID\":18,\"PhysicalTableID\":158,\"TableInfo\":{\"id\":158,\"name\":{\"O\":\"example17\",\"L\":\"example17\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805887381539,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example17\",\"TableID\":158,\"IsPartition\":false},\"Version\":449527805887381544,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":18}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449527814144917523,\"HandleKey\":18}"] [2024/05/04 16:48:45.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805965762575,\"CommitTs\":449527805965762577,\"RowID\":1,\"PhysicalTableID\":162,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805952917527,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":162,\"name\":{\"O\":\"pBefore1950\",\"L\":\"pbefore1950\"},\"less_than\":[\"1950\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":163,\"name\":{\"O\":\"p1950\",\"L\":\"p1950\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":166,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":167,\"name\":{\"O\":\"p1990\",\"L\":\"p1990\"},\"less_than\":[\"2000\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":6,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449527805952917532,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449527814669205514,\"HandleKey\":1}"] [2024/05/04 16:48:45.524 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805965762575,\"CommitTs\":449527805965762577,\"RowID\":1977,\"PhysicalTableID\":165,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805952917527,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":162,\"name\":{\"O\":\"pBefore1950\",\"L\":\"pbefore1950\"},\"less_than\":[\"1950\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":163,\"name\":{\"O\":\"p1950\",\"L\":\"p1950\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":166,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":167,\"name\":{\"O\":\"p1990\",\"L\":\"p1990\"},\"less_than\":[\"2000\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":6,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449527805952917532,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1977}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449527814669205517,\"HandleKey\":1977}"] [2024/05/04 16:48:46.524 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527805965762575,\"CommitTs\":449527805965762577,\"RowID\":1999,\"PhysicalTableID\":167,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527805952917527,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":162,\"name\":{\"O\":\"pBefore1950\",\"L\":\"pbefore1950\"},\"less_than\":[\"1950\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":163,\"name\":{\"O\":\"p1950\",\"L\":\"p1950\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":166,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":167,\"name\":{\"O\":\"p1990\",\"L\":\"p1990\"},\"less_than\":[\"2000\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":6,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449527805952917532,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1999}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449527814918242310,\"HandleKey\":1999}"] [2024/05/04 16:48:52.524 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527806162632721,\"CommitTs\":449527806162632722,\"RowID\":2003,\"PhysicalTableID\":174,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":null,\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527806149525512,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":null,\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":169,\"name\":{\"O\":\"pBefore1960\",\"L\":\"pbefore1960\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":172,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":174,\"name\":{\"O\":\"p1990to2010\",\"L\":\"p1990to2010\"},\"less_than\":[\"2010\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":null,\"dropping_definitions\":null,\"NewPartitionIDs\":null,\"states\":null,\"num\":5,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":1,\"ddl_expr\":\"`id`\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449527806149525517,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":2003}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449527816491106309,\"HandleKey\":2003}"] [2024/05/04 16:48:56.523 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449527807879675919,\"CommitTs\":449527807879675920,\"RowID\":1,\"PhysicalTableID\":182,\"TableInfo\":{\"id\":182,\"name\":{\"O\":\"example19\",\"L\":\"example19\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449527807866306591,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example19\",\"TableID\":182,\"IsPartition\":false},\"Version\":449527807866306595,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449527817539420169,\"HandleKey\":1}"] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:49:03 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 28.59 secs (130468416 bytes/sec) [Pipeline] { [Pipeline] cache =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 7.32 secs (509469661 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2cf6a40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z, pid:4944, start at 2024-05-04 16:49:26.955797476 +0800 CST m=+5.213368669 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:51:26.965 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:49:26.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:39:26.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2cf6a40004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z, pid:4944, start at 2024-05-04 16:49:26.955797476 +0800 CST m=+5.213368669 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:51:26.965 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:49:26.953 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:39:26.953 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2cf9b0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lz6-1j34z, pid:5028, start at 2024-05-04 16:49:27.162195768 +0800 CST m=+5.371844365 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:51:27.169 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:49:27.148 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:39:27.148 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/storage_csv_update/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/storage_csv_update/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/storage_csv_update/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sat May 4 16:49:32 CST 2024] <<<<<< START cdc server in storage_csv_update case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.63206322.out server --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/storage_csv_update/cdc_data --cluster-id default + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:49:35 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d541e0c-163e-470d-aba1-5fc39d2a8fde {"id":"8d541e0c-163e-470d-aba1-5fc39d2a8fde","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812572} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cb1566cd 8d541e0c-163e-470d-aba1-5fc39d2a8fde /tidb/cdc/default/default/upstream/7365063857669827279 {"id":7365063857669827279,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d541e0c-163e-470d-aba1-5fc39d2a8fde {"id":"8d541e0c-163e-470d-aba1-5fc39d2a8fde","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812572} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cb1566cd 8d541e0c-163e-470d-aba1-5fc39d2a8fde /tidb/cdc/default/default/upstream/7365063857669827279 {"id":7365063857669827279,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d541e0c-163e-470d-aba1-5fc39d2a8fde {"id":"8d541e0c-163e-470d-aba1-5fc39d2a8fde","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812572} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cb1566cd 8d541e0c-163e-470d-aba1-5fc39d2a8fde /tidb/cdc/default/default/upstream/7365063857669827279 {"id":7365063857669827279,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.cli.6376.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml Create changefeed successfully! ID: 3131ec22-be5b-4a67-8c8f-189cbe007836 Info: {"upstream_id":7365063857669827279,"namespace":"default","id":"3131ec22-be5b-4a67-8c8f-189cbe007836","sink_uri":"file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:49:35.57762854+08:00","start_ts":449527827792658433,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527827792658433,"checkpoint_ts":449527827792658433,"checkpoint_time":"2024-05-04 16:49:35.503"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 13.15 secs (283679432 bytes/sec) [Pipeline] { [Pipeline] cache Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 10.50 secs (355177830 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=3b1a2f6f-d6c8-4c0c-ada6-2259e4119c37 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-lsdkl GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-lsdkl pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=c9e39ee4-7c23-45bb-ba78-bebf80f92e62 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-bpq0z GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-bpq0z pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G03 Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=fdbcfe82-fa4c-4dd0-884b-4c0d05d74230 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-w70sv GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw pingcap_tiflow_pull_cdc_integration_storage_test_1781-w70sv GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G02 Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=ce71c547-c045-46b8-832c-a86468ea6577 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-pqsx2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f pingcap_tiflow_pull_cdc_integration_storage_test_1781-pqsx2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:49:51 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=a105832f-8e14-44f1-b1f1-23e7c9104d75 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-qv0bl GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-qv0bl pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: storage... <<================= [Pipeline] sh The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=e447cec0-b4df-4ff6-855e-445962c750d2 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-80lm2 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj pingcap_tiflow_pull_cdc_integration_storage_test_1781-80lm2 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=f1ae91bb-50d9-4366-a12e-fa90c66aa5af BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-jjg7q GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-jjg7q pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc already exists) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G04 [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=67087e36-3b27-4cfe-a377-45979cfcc49b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-7svrs GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7 pingcap_tiflow_pull_cdc_integration_storage_test_1781-7svrs GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] // cache [Pipeline] } [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] // dir [Pipeline] } [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] // withCredentials [Pipeline] } [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: storage... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend [Pipeline] { Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide Verifying downstream PD is started... [Pipeline] { [Pipeline] // timeout [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // stage [Pipeline] } [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC3CF8C8A72FF0 < X-Xss-Protection: 1; mode=block < Date: Sat, 04 May 2024 08:49:57 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Sat May 4 16:49:57 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:50:03 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f4a700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs, pid:1273, start at 2024-05-04 16:50:05.122574435 +0800 CST m=+5.494699872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:05.131 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:05.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:05.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f4d3c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d, pid:1300, start at 2024-05-04 16:50:05.275080311 +0800 CST m=+5.219960168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:05.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:05.263 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:05.263 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:50:06 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f4a700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs, pid:1273, start at 2024-05-04 16:50:05.122574435 +0800 CST m=+5.494699872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:05.131 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:05.133 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:05.133 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f4a740015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs, pid:1356, start at 2024-05-04 16:50:05.122542304 +0800 CST m=+5.438338275 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:05.130 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:05.134 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:05.134 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f4d3c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d, pid:1300, start at 2024-05-04 16:50:05.275080311 +0800 CST m=+5.219960168 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:05.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:05.263 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:05.263 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f4d240005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d, pid:1373, start at 2024-05-04 16:50:05.259871588 +0800 CST m=+5.129855228 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:05.269 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:05.257 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:05.257 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 9.37 secs (398211625 bytes/sec) [Pipeline] { [Pipeline] cache ls /tmp/tidb_cdc_test/storage_csv_update/storage_test/test/test_update 449527828212088844 run task successfully start check update + workdir=/tmp/tidb_cdc_test/storage_csv_update + sink_uri='file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/storage_csv_update ++ date + echo '[Sat May 4 16:50:09 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>>' [Sat May 4 16:50:09 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>> + cd /tmp/tidb_cdc_test/storage_csv_update + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/storage_csv_update + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f7f00001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj, pid:1298, start at 2024-05-04 16:50:08.494531964 +0800 CST m=+5.166139597 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:08.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:08.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:08.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f7f00001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj, pid:1298, start at 2024-05-04 16:50:08.494531964 +0800 CST m=+5.166139597 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:08.501 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:08.498 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:08.498 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f80840015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj, pid:1377, start at 2024-05-04 16:50:08.580123332 +0800 CST m=+5.197960130 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:08.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:08.545 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:08.545 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f62700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw, pid:1240, start at 2024-05-04 16:50:06.654993313 +0800 CST m=+5.235121281 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:06.661 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:06.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:06.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f62700019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw, pid:1240, start at 2024-05-04 16:50:06.654993313 +0800 CST m=+5.235121281 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:06.661 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:06.620 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:06.620 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f63e40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw, pid:1326, start at 2024-05-04 16:50:06.730977015 +0800 CST m=+5.232291196 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:06.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:06.713 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:06.713 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f6d840005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:1245, start at 2024-05-04 16:50:07.332873781 +0800 CST m=+5.675148241 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:07.340 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:07.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:07.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f6d840005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:1245, start at 2024-05-04 16:50:07.332873781 +0800 CST m=+5.675148241 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:07.340 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:07.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:07.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f6f08000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:1318, start at 2024-05-04 16:50:07.438408789 +0800 CST m=+5.721922863 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:07.445 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:07.426 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:07.426 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2622.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f8cfc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr, pid:1232, start at 2024-05-04 16:50:09.368968992 +0800 CST m=+5.287313627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:09.375 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:09.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:09.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 16:50:11 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26192621.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Sat May 4 16:50:11 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.26612663.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2631.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449527836910288897 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527836910288897 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "readproportion"="0" "recordcount"="10" "requestdistribution"="uniform" "readallfields"="true" "workload"="core" "mysql.port"="4000" "updateproportion"="0" "insertproportion"="0" "mysql.db"="sink_retry" "operationcount"="0" "mysql.user"="root" "threadcount"="2" "mysql.host"="127.0.0.1" "scanproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 8.41326ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2139.3, Avg(us): 1597, Min(us): 1017, Max(us): 3595, 95th(us): 4000, 99th(us): 4000 [Sat May 4 16:50:12 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.26742676.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f8cfc0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr, pid:1232, start at 2024-05-04 16:50:09.368968992 +0800 CST m=+5.287313627 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:09.375 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:09.343 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:09.343 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f8e780007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr, pid:1322, start at 2024-05-04 16:50:09.444545276 +0800 CST m=+5.308509592 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:09.451 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:09.438 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:09.438 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f8cdc000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7, pid:1172, start at 2024-05-04 16:50:09.348707224 +0800 CST m=+5.250334585 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:09.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:09.335 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:09.335 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f8cdc000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7, pid:1172, start at 2024-05-04 16:50:09.348707224 +0800 CST m=+5.250334585 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:09.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:09.335 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:09.335 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b2f8e9c000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7, pid:1254, start at 2024-05-04 16:50:09.456357938 +0800 CST m=+5.287564245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:09.465 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:09.447 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:09.447 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449527837378478082 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527837378478082 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 16:50:13 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26672669.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:50:13 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 16:50:14 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.26272629.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2634.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:50:14 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/82833a73-509e-41f8-9c37-f49e868cc602 {"id":"82833a73-509e-41f8-9c37-f49e868cc602","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812611} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbb3c8ca 82833a73-509e-41f8-9c37-f49e868cc602 /tidb/cdc/default/default/upstream/7365064024829717359 {"id":7365064024829717359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/82833a73-509e-41f8-9c37-f49e868cc602 {"id":"82833a73-509e-41f8-9c37-f49e868cc602","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812611} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbb3c8ca 82833a73-509e-41f8-9c37-f49e868cc602 /tidb/cdc/default/default/upstream/7365064024829717359 {"id":7365064024829717359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/82833a73-509e-41f8-9c37-f49e868cc602 {"id":"82833a73-509e-41f8-9c37-f49e868cc602","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812611} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbb3c8ca 82833a73-509e-41f8-9c37-f49e868cc602 /tidb/cdc/default/default/upstream/7365064024829717359 {"id":7365064024829717359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:50:14 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f18d72fa-c951-476f-9537-f5cfe7f73be2 {"id":"f18d72fa-c951-476f-9537-f5cfe7f73be2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812611} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cba919cd f18d72fa-c951-476f-9537-f5cfe7f73be2 /tidb/cdc/default/default/upstream/7365064018802898904 {"id":7365064018802898904,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f18d72fa-c951-476f-9537-f5cfe7f73be2 {"id":"f18d72fa-c951-476f-9537-f5cfe7f73be2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812611} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cba919cd f18d72fa-c951-476f-9537-f5cfe7f73be2 /tidb/cdc/default/default/upstream/7365064018802898904 {"id":7365064018802898904,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f18d72fa-c951-476f-9537-f5cfe7f73be2 {"id":"f18d72fa-c951-476f-9537-f5cfe7f73be2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812611} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cba919cd f18d72fa-c951-476f-9537-f5cfe7f73be2 /tidb/cdc/default/default/upstream/7365064018802898904 {"id":7365064018802898904,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: abe3f6da-382d-41f5-8f7e-b01cca5687b0 Info: {"upstream_id":7365064018802898904,"namespace":"default","id":"abe3f6da-382d-41f5-8f7e-b01cca5687b0","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-04T16:50:14.823598923+08:00","start_ts":449527837232463873,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527837232463873,"checkpoint_ts":449527837232463873,"checkpoint_time":"2024-05-04 16:50:11.513"} + workdir=/tmp/tidb_cdc_test/charset_gbk + sink_uri=mysql://normal:123456@127.0.0.1:3306/ + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/charset_gbk ++ date + echo '[Sat May 4 16:50:14 CST 2024] <<<<<< START storage consumer in charset_gbk case >>>>>>' [Sat May 4 16:50:14 CST 2024] <<<<<< START storage consumer in charset_gbk case >>>>>> + cd /tmp/tidb_cdc_test/charset_gbk + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/charset_gbk + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/charset_gbk/cdc_storage_consumer.log --log-level debug --upstream-uri mysql://normal:123456@127.0.0.1:3306/ --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + workdir=/tmp/tidb_cdc_test/kv_client_stream_reconnect + sink_uri='file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/storage_test/kv-client-stream-reconnect-15795?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/kv_client_stream_reconnect ++ date + echo '[Sat May 4 16:50:14 CST 2024] <<<<<< START storage consumer in kv_client_stream_reconnect case >>>>>>' [Sat May 4 16:50:14 CST 2024] <<<<<< START storage consumer in kv_client_stream_reconnect case >>>>>> + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/storage_test/kv-client-stream-reconnect-15795?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:50:15 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8cc9128-0bce-45c3-a38d-a684d311c23f {"id":"f8cc9128-0bce-45c3-a38d-a684d311c23f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbab53ee f8cc9128-0bce-45c3-a38d-a684d311c23f /tidb/cdc/default/default/upstream/7365064018668350231 {"id":7365064018668350231,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8cc9128-0bce-45c3-a38d-a684d311c23f {"id":"f8cc9128-0bce-45c3-a38d-a684d311c23f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbab53ee f8cc9128-0bce-45c3-a38d-a684d311c23f /tidb/cdc/default/default/upstream/7365064018668350231 {"id":7365064018668350231,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f8cc9128-0bce-45c3-a38d-a684d311c23f {"id":"f8cc9128-0bce-45c3-a38d-a684d311c23f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812612} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbab53ee f8cc9128-0bce-45c3-a38d-a684d311c23f /tidb/cdc/default/default/upstream/7365064018668350231 {"id":7365064018668350231,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2730.out cli changefeed create --start-ts=449527836910288897 '--sink-uri=file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-2545?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 325d344e-d0fc-4b84-a9ee-8ba53736b80f Info: {"upstream_id":7365064018668350231,"namespace":"default","id":"325d344e-d0fc-4b84-a9ee-8ba53736b80f","sink_uri":"file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-2545?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:50:15.563148948+08:00","start_ts":449527836910288897,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527836910288897,"checkpoint_ts":449527836910288897,"checkpoint_time":"2024-05-04 16:50:10.284"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + tso='449527838090199042 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527838090199042 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 16:50:16 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.26772679.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:50:16 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f95fbe1f-6d50-48bc-a73b-609ef1d02676 {"id":"f95fbe1f-6d50-48bc-a73b-609ef1d02676","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812613} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbabaadc f95fbe1f-6d50-48bc-a73b-609ef1d02676 /tidb/cdc/default/default/upstream/7365064014517843450 {"id":7365064014517843450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f95fbe1f-6d50-48bc-a73b-609ef1d02676 {"id":"f95fbe1f-6d50-48bc-a73b-609ef1d02676","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812613} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbabaadc f95fbe1f-6d50-48bc-a73b-609ef1d02676 /tidb/cdc/default/default/upstream/7365064014517843450 {"id":7365064014517843450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f95fbe1f-6d50-48bc-a73b-609ef1d02676 {"id":"f95fbe1f-6d50-48bc-a73b-609ef1d02676","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812613} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbabaadc f95fbe1f-6d50-48bc-a73b-609ef1d02676 /tidb/cdc/default/default/upstream/7365064014517843450 {"id":7365064014517843450,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2725.out cli changefeed create --start-ts=449527837378478082 '--sink-uri=file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-7612?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 3313cffc-42a6-4b8d-a85f-8255ecbe9088 Info: {"upstream_id":7365064014517843450,"namespace":"default","id":"3313cffc-42a6-4b8d-a85f-8255ecbe9088","sink_uri":"file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-7612?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:50:17.040168822+08:00","start_ts":449527837378478082,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527837378478082,"checkpoint_ts":449527837378478082,"checkpoint_time":"2024-05-04 16:50:12.070"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + workdir=/tmp/tidb_cdc_test/sink_retry + sink_uri='file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-2545?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/sink_retry ++ date + echo '[Sat May 4 16:50:17 CST 2024] <<<<<< START storage consumer in sink_retry case >>>>>>' [Sat May 4 16:50:17 CST 2024] <<<<<< START storage consumer in sink_retry case >>>>>> + cd /tmp/tidb_cdc_test/sink_retry + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/sink_retry + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/sink_retry/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-2545?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:50:17 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/006b0ab8-0426-43d1-ba37-1e5400b87f0b {"id":"006b0ab8-0426-43d1-ba37-1e5400b87f0b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812614} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbb6e3cc 006b0ab8-0426-43d1-ba37-1e5400b87f0b /tidb/cdc/default/default/upstream/7365064037826052617 {"id":7365064037826052617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/006b0ab8-0426-43d1-ba37-1e5400b87f0b {"id":"006b0ab8-0426-43d1-ba37-1e5400b87f0b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812614} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbb6e3cc 006b0ab8-0426-43d1-ba37-1e5400b87f0b /tidb/cdc/default/default/upstream/7365064037826052617 {"id":7365064037826052617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/006b0ab8-0426-43d1-ba37-1e5400b87f0b {"id":"006b0ab8-0426-43d1-ba37-1e5400b87f0b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812614} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbb6e3cc 006b0ab8-0426-43d1-ba37-1e5400b87f0b /tidb/cdc/default/default/upstream/7365064037826052617 {"id":7365064037826052617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/gc_safepoint + sink_uri='file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-26273?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/gc_safepoint ++ date + echo '[Sat May 4 16:50:17 CST 2024] <<<<<< START storage consumer in gc_safepoint case >>>>>>' [Sat May 4 16:50:17 CST 2024] <<<<<< START storage consumer in gc_safepoint case >>>>>> + cd /tmp/tidb_cdc_test/gc_safepoint + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/gc_safepoint + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-26273?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' 0 check diff failed 1-th time, retry later + set +x + workdir=/tmp/tidb_cdc_test/row_format + sink_uri='file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-7612?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/row_format ++ date + echo '[Sat May 4 16:50:18 CST 2024] <<<<<< START storage consumer in row_format case >>>>>>' [Sat May 4 16:50:18 CST 2024] <<<<<< START storage consumer in row_format case >>>>>> + cd /tmp/tidb_cdc_test/row_format + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/row_format + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/row_format/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-7612?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: storage... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide check diff failed 2-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:50:19 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d634ab7d-573b-49ff-b4ef-dd6d680a5169 {"id":"d634ab7d-573b-49ff-b4ef-dd6d680a5169","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812616} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbb8d1d5 d634ab7d-573b-49ff-b4ef-dd6d680a5169 /tidb/cdc/default/default/upstream/7365064038972729256 {"id":7365064038972729256,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d634ab7d-573b-49ff-b4ef-dd6d680a5169 {"id":"d634ab7d-573b-49ff-b4ef-dd6d680a5169","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812616} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbb8d1d5 d634ab7d-573b-49ff-b4ef-dd6d680a5169 /tidb/cdc/default/default/upstream/7365064038972729256 {"id":7365064038972729256,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d634ab7d-573b-49ff-b4ef-dd6d680a5169 {"id":"d634ab7d-573b-49ff-b4ef-dd6d680a5169","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812616} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbb8d1d5 d634ab7d-573b-49ff-b4ef-dd6d680a5169 /tidb/cdc/default/default/upstream/7365064038972729256 {"id":7365064038972729256,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2736.out cli changefeed create --start-ts=449527838090199042 '--sink-uri=file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-4054?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: a715c3ce-b463-4550-9fda-0ae6e48ac235 Info: {"upstream_id":7365064038972729256,"namespace":"default","id":"a715c3ce-b463-4550-9fda-0ae6e48ac235","sink_uri":"file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-4054?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:50:19.788367843+08:00","start_ts":449527838090199042,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527838090199042,"checkpoint_ts":449527838090199042,"checkpoint_time":"2024-05-04 16:50:14.785"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Sat May 4 16:50:19 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.27832785.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC3CFE21926B5E < X-Xss-Protection: 1; mode=block < Date: Sat, 04 May 2024 08:50:20 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Sat May 4 16:50:20 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT + set +x + workdir=/tmp/tidb_cdc_test/foreign_key + sink_uri='file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-4054?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/foreign_key ++ date + echo '[Sat May 4 16:50:21 CST 2024] <<<<<< START storage consumer in foreign_key case >>>>>>' [Sat May 4 16:50:21 CST 2024] <<<<<< START storage consumer in foreign_key case >>>>>> + cd /tmp/tidb_cdc_test/foreign_key + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/foreign_key + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/foreign_key/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-4054?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check diff successfully [Sat May 4 16:50:21 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:50:22 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b5da311-d7f5-4db0-a48a-ea2beb653477 {"id":"8b5da311-d7f5-4db0-a48a-ea2beb653477","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812619} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbaf3886 8b5da311-d7f5-4db0-a48a-ea2beb653477 /tidb/cdc/default/default/upstream/7365064019432166890 {"id":7365064019432166890,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b5da311-d7f5-4db0-a48a-ea2beb653477 {"id":"8b5da311-d7f5-4db0-a48a-ea2beb653477","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812619} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbaf3886 8b5da311-d7f5-4db0-a48a-ea2beb653477 /tidb/cdc/default/default/upstream/7365064019432166890 {"id":7365064019432166890,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8b5da311-d7f5-4db0-a48a-ea2beb653477 {"id":"8b5da311-d7f5-4db0-a48a-ea2beb653477","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812619} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cbaf3886 8b5da311-d7f5-4db0-a48a-ea2beb653477 /tidb/cdc/default/default/upstream/7365064019432166890 {"id":7365064019432166890,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/processor_err_chan + sink_uri='file:///tmp/tidb_cdc_test/processor_err_chan/storage_test/ticdc-processor-err-chan-12455?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_err_chan ++ date + echo '[Sat May 4 16:50:22 CST 2024] <<<<<< START storage consumer in processor_err_chan case >>>>>>' [Sat May 4 16:50:22 CST 2024] <<<<<< START storage consumer in processor_err_chan case >>>>>> + cd /tmp/tidb_cdc_test/processor_err_chan + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_err_chan + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_err_chan/storage_test/ticdc-processor-err-chan-12455?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_state http://127.0.0.1:2379 46d6ff0e-290f-492e-b42a-7c82e6aded9c normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=46d6ff0e-290f-492e-b42a-7c82e6aded9c + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 46d6ff0e-290f-492e-b42a-7c82e6aded9c -s check diff failed 3-th time, retry later + info='{ "upstream_id": 7365064019432166890, "namespace": "default", "id": "46d6ff0e-290f-492e-b42a-7c82e6aded9c", "state": "normal", "checkpoint_tso": 449527840067813383, "checkpoint_time": "2024-05-04 16:50:22.329", "error": null }' + echo '{ "upstream_id": 7365064019432166890, "namespace": "default", "id": "46d6ff0e-290f-492e-b42a-7c82e6aded9c", "state": "normal", "checkpoint_tso": 449527840067813383, "checkpoint_time": "2024-05-04 16:50:22.329", "error": null }' { "upstream_id": 7365064019432166890, "namespace": "default", "id": "46d6ff0e-290f-492e-b42a-7c82e6aded9c", "state": "normal", "checkpoint_tso": 449527840067813383, "checkpoint_time": "2024-05-04 16:50:22.329", "error": null } ++ echo '{' '"upstream_id":' 7365064019432166890, '"namespace":' '"default",' '"id":' '"46d6ff0e-290f-492e-b42a-7c82e6aded9c",' '"state":' '"normal",' '"checkpoint_tso":' 449527840067813383, '"checkpoint_time":' '"2024-05-04' '16:50:22.329",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365064019432166890, '"namespace":' '"default",' '"id":' '"46d6ff0e-290f-492e-b42a-7c82e6aded9c",' '"state":' '"normal",' '"checkpoint_tso":' 449527840067813383, '"checkpoint_time":' '"2024-05-04' '16:50:22.329",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 not exists for 1-th check, retry later check diff failed 1-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 11.62 secs (321164145 bytes/sec) [Pipeline] { [Pipeline] cache table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later check diff failed 4-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:50:23 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table test.finish_mark not exists for 2-th check, retry later check diff failed 5-th time, retry later check diff failed 3-th time, retry later table test.finish_mark not exists for 3-th check, retry later check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365064037826052617 449527841329774595 449527838826299398 run task successfully check_changefeed_state http://127.0.0.1:2379 c2e9ee67-d431-4ef3-be55-dc361fb84262 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=c2e9ee67-d431-4ef3-be55-dc361fb84262 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c2e9ee67-d431-4ef3-be55-dc361fb84262 -s table foreign_key.finish_mark not exists for 1-th check, retry later + info='{ "upstream_id": 7365064037826052617, "namespace": "default", "id": "c2e9ee67-d431-4ef3-be55-dc361fb84262", "state": "stopped", "checkpoint_tso": 449527841854062596, "checkpoint_time": "2024-05-04 16:50:29.143", "error": null }' + echo '{ "upstream_id": 7365064037826052617, "namespace": "default", "id": "c2e9ee67-d431-4ef3-be55-dc361fb84262", "state": "stopped", "checkpoint_tso": 449527841854062596, "checkpoint_time": "2024-05-04 16:50:29.143", "error": null }' { "upstream_id": 7365064037826052617, "namespace": "default", "id": "c2e9ee67-d431-4ef3-be55-dc361fb84262", "state": "stopped", "checkpoint_tso": 449527841854062596, "checkpoint_time": "2024-05-04 16:50:29.143", "error": null } ++ echo '{' '"upstream_id":' 7365064037826052617, '"namespace":' '"default",' '"id":' '"c2e9ee67-d431-4ef3-be55-dc361fb84262",' '"state":' '"stopped",' '"checkpoint_tso":' 449527841854062596, '"checkpoint_time":' '"2024-05-04' '16:50:29.143",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365064037826052617, '"namespace":' '"default",' '"id":' '"c2e9ee67-d431-4ef3-be55-dc361fb84262",' '"state":' '"stopped",' '"checkpoint_tso":' 449527841854062596, '"checkpoint_time":' '"2024-05-04' '16:50:29.143",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365064037826052617 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:50:29 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> check diff failed 4-th time, retry later table test.finish_mark not exists for 4-th check, retry later safepoint is unexpected forward: 449527841591918596 449527838826299398 -> 449527841854062595 449527838826299398 run task failed 1-th time, retry later table foreign_key.finish_mark not exists for 2-th check, retry later check diff failed 5-th time, retry later table test.finish_mark not exists for 5-th check, retry later check_safepoint_equal http://127.0.0.1:2379 7365064037826052617 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:50:32 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> table foreign_key.finish_mark not exists for 3-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 8.09 secs (461332691 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 6-th time, retry later table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table foreign_key.finish_mark not exists for 4-th check, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 c2e9ee67-d431-4ef3-be55-dc361fb84262 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=c2e9ee67-d431-4ef3-be55-dc361fb84262 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c2e9ee67-d431-4ef3-be55-dc361fb84262 -s find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:50:36 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> check diff successfully table row_format.finish_mark not exists for 1-th check, retry later table row_format.finish_mark not exists for 2-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:50:36 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> + info='{ "upstream_id": 7365064037826052617, "namespace": "default", "id": "c2e9ee67-d431-4ef3-be55-dc361fb84262", "state": "normal", "checkpoint_tso": 449527841854062596, "checkpoint_time": "2024-05-04 16:50:29.143", "error": null }' + echo '{ "upstream_id": 7365064037826052617, "namespace": "default", "id": "c2e9ee67-d431-4ef3-be55-dc361fb84262", "state": "normal", "checkpoint_tso": 449527841854062596, "checkpoint_time": "2024-05-04 16:50:29.143", "error": null }' { "upstream_id": 7365064037826052617, "namespace": "default", "id": "c2e9ee67-d431-4ef3-be55-dc361fb84262", "state": "normal", "checkpoint_tso": 449527841854062596, "checkpoint_time": "2024-05-04 16:50:29.143", "error": null } ++ echo '{' '"upstream_id":' 7365064037826052617, '"namespace":' '"default",' '"id":' '"c2e9ee67-d431-4ef3-be55-dc361fb84262",' '"state":' '"normal",' '"checkpoint_tso":' 449527841854062596, '"checkpoint_time":' '"2024-05-04' '16:50:29.143",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365064037826052617, '"namespace":' '"default",' '"id":' '"c2e9ee67-d431-4ef3-be55-dc361fb84262",' '"state":' '"normal",' '"checkpoint_tso":' 449527841854062596, '"checkpoint_time":' '"2024-05-04' '16:50:29.143",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365064037826052617 449527841854062595 449527841854062596 \033[0;36m<<< Run all test success >>>\033[0m wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table row_format.finish_mark not exists for 3-th check, retry later wait process cdc.test exit for 3-th time... table foreign_key.finish_mark not exists for 5-th check, retry later run task successfully check_changefeed_state http://127.0.0.1:2379 c2e9ee67-d431-4ef3-be55-dc361fb84262 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=c2e9ee67-d431-4ef3-be55-dc361fb84262 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c c2e9ee67-d431-4ef3-be55-dc361fb84262 -s cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:50:38 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> + info='{ "upstream_id": 7365064037826052617, "namespace": "default", "id": "c2e9ee67-d431-4ef3-be55-dc361fb84262", "state": "stopped", "checkpoint_tso": 449527841854062596, "checkpoint_time": "2024-05-04 16:50:29.143", "error": null }' + echo '{ "upstream_id": 7365064037826052617, "namespace": "default", "id": "c2e9ee67-d431-4ef3-be55-dc361fb84262", "state": "stopped", "checkpoint_tso": 449527841854062596, "checkpoint_time": "2024-05-04 16:50:29.143", "error": null }' { "upstream_id": 7365064037826052617, "namespace": "default", "id": "c2e9ee67-d431-4ef3-be55-dc361fb84262", "state": "stopped", "checkpoint_tso": 449527841854062596, "checkpoint_time": "2024-05-04 16:50:29.143", "error": null } ++ echo '{' '"upstream_id":' 7365064037826052617, '"namespace":' '"default",' '"id":' '"c2e9ee67-d431-4ef3-be55-dc361fb84262",' '"state":' '"stopped",' '"checkpoint_tso":' 449527841854062596, '"checkpoint_time":' '"2024-05-04' '16:50:29.143",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365064037826052617, '"namespace":' '"default",' '"id":' '"c2e9ee67-d431-4ef3-be55-dc361fb84262",' '"state":' '"stopped",' '"checkpoint_tso":' 449527841854062596, '"checkpoint_time":' '"2024-05-04' '16:50:29.143",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 2c3486df-cb28-4d87-a97d-a5f3dd92a221 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=2c3486df-cb28-4d87-a97d-a5f3dd92a221 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 2c3486df-cb28-4d87-a97d-a5f3dd92a221 -s + info='{ "upstream_id": 7365064037826052617, "namespace": "default", "id": "2c3486df-cb28-4d87-a97d-a5f3dd92a221", "state": "normal", "checkpoint_tso": 449527844226465796, "checkpoint_time": "2024-05-04 16:50:38.193", "error": null }' + echo '{ "upstream_id": 7365064037826052617, "namespace": "default", "id": "2c3486df-cb28-4d87-a97d-a5f3dd92a221", "state": "normal", "checkpoint_tso": 449527844226465796, "checkpoint_time": "2024-05-04 16:50:38.193", "error": null }' { "upstream_id": 7365064037826052617, "namespace": "default", "id": "2c3486df-cb28-4d87-a97d-a5f3dd92a221", "state": "normal", "checkpoint_tso": 449527844226465796, "checkpoint_time": "2024-05-04 16:50:38.193", "error": null } ++ echo '{' '"upstream_id":' 7365064037826052617, '"namespace":' '"default",' '"id":' '"2c3486df-cb28-4d87-a97d-a5f3dd92a221",' '"state":' '"normal",' '"checkpoint_tso":' 449527844226465796, '"checkpoint_time":' '"2024-05-04' '16:50:38.193",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365064037826052617, '"namespace":' '"default",' '"id":' '"2c3486df-cb28-4d87-a97d-a5f3dd92a221",' '"state":' '"normal",' '"checkpoint_tso":' 449527844226465796, '"checkpoint_time":' '"2024-05-04' '16:50:38.193",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365064037826052617 table row_format.finish_mark not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 6-th check, retry later run task successfully table row_format.finish_mark not exists for 5-th check, retry later Changefeed remove successfully. ID: c2e9ee67-d431-4ef3-be55-dc361fb84262 CheckpointTs: 449527841854062596 SinkURI: file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-26273?protocol=canal-json&enable-tidb-extension=true check_safepoint_forward http://127.0.0.1:2379 7365064037826052617 449527841854062595 449527844226465796 449527841854062596 table foreign_key.finish_mark not exists for 7-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=storage + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run storage + '[' storage '!=' storage ']' + rm -rf /tmp/tidb_cdc_test/storage_cleanup + mkdir -p /tmp/tidb_cdc_test/storage_cleanup + start_tidb_cluster --workdir /tmp/tidb_cdc_test/storage_cleanup The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 7.22 secs (516947748 bytes/sec) [Pipeline] { [Pipeline] cache run task successfully table row_format.finish_mark not exists for 6-th check, retry later table foreign_key.finish_mark not exists for 8-th check, retry later Changefeed remove successfully. ID: 2c3486df-cb28-4d87-a97d-a5f3dd92a221 CheckpointTs: 449527845524078598 SinkURI: file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-26273?protocol=canal-json&enable-tidb-extension=true check_safepoint_cleared http://127.0.0.1:2379 7365064037826052617 run task successfully wait process cdc.test exit for 1-th time... table row_format.finish_mark not exists for 7-th check, retry later wait process cdc.test exit for 2-th time... table foreign_key.finish_mark not exists for 9-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/storage_cleanup Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:50:46 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table row_format.finish_mark not exists for 8-th check, retry later table foreign_key.finish_mark not exists for 10-th check, retry later table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "mysql.user"="root" "requestdistribution"="uniform" "readallfields"="true" "threadcount"="2" "dotransactions"="false" "scanproportion"="0" "insertproportion"="0" "operationcount"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" "recordcount"="10" "mysql.port"="4000" "workload"="core" "readproportion"="0" "mysql.db"="sink_retry" ********************************************** Run finished, takes 3.956015ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3518.5, Avg(us): 693, Min(us): 480, Max(us): 1581, 95th(us): 2000, 99th(us): 2000 start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table row_format.finish_mark not exists for 9-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table foreign_key.finish_mark not exists for 11-th check, retry later Verifying downstream PD is started... table row_format.finish_mark not exists for 10-th check, retry later table foreign_key.finish_mark not exists for 12-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 1-th time, retry later table row_format.finish_mark not exists for 11-th check, retry later table foreign_key.finish_mark not exists for 13-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later table row_format.finish_mark not exists for 12-th check, retry later table foreign_key.finish_mark not exists for 14-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table row_format.finish_mark not exists for 13-th check, retry later table foreign_key.finish_mark not exists for 15-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b327e94000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f, pid:2298, start at 2024-05-04 16:50:57.589512187 +0800 CST m=+5.271717249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:57.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:57.573 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:57.573 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later table row_format.finish_mark not exists for 14-th check, retry later table foreign_key.finish_mark not exists for 16-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b327e94000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f, pid:2298, start at 2024-05-04 16:50:57.589512187 +0800 CST m=+5.271717249 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:57.600 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:57.573 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:57.573 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3280100016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f, pid:2381, start at 2024-05-04 16:50:57.706210209 +0800 CST m=+5.328885997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:52:57.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:50:57.718 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:40:57.718 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/storage_cleanup/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/storage_cleanup/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/storage_cleanup/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/storage_cleanup/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/storage_cleanup/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table row_format.finish_mark not exists for 15-th check, retry later table foreign_key.finish_mark not exists for 17-th check, retry later + cd /tmp/tidb_cdc_test/storage_cleanup + run_cdc_server --workdir /tmp/tidb_cdc_test/storage_cleanup --binary cdc.test [Sat May 4 16:51:02 CST 2024] <<<<<< START cdc server in storage_cleanup case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_cleanup.36473649.out server --log-file /tmp/tidb_cdc_test/storage_cleanup/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/storage_cleanup/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b32d0f80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d, pid:3887, start at 2024-05-04 16:51:02.858958342 +0800 CST m=+6.073553642 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:02.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:02.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:02.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 16-th check, retry later table foreign_key.finish_mark exists check diff successfully check diff failed 6-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b32d0f80007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d, pid:3887, start at 2024-05-04 16:51:02.858958342 +0800 CST m=+6.073553642 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:02.870 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:02.846 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:02.846 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b32d0b00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-lsd-s9j1d, pid:3959, start at 2024-05-04 16:51:02.875173275 +0800 CST m=+5.730258380 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:02.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:02.878 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:02.878 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b32e7d00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:4276, start at 2024-05-04 16:51:04.338789044 +0800 CST m=+5.241706685 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:04.347 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:04.308 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:04.308 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:51:05 CST 2024] <<<<<< run test case foreign_key success! >>>>>> table row_format.finish_mark not exists for 17-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:05 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6da06ee9-2891-4b79-a426-bc8fc6cd25a1 {"id":"6da06ee9-2891-4b79-a426-bc8fc6cd25a1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812663} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc734ace 6da06ee9-2891-4b79-a426-bc8fc6cd25a1 /tidb/cdc/default/default/upstream/7365064247257976539 {"id":7365064247257976539,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6da06ee9-2891-4b79-a426-bc8fc6cd25a1 {"id":"6da06ee9-2891-4b79-a426-bc8fc6cd25a1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812663} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc734ace 6da06ee9-2891-4b79-a426-bc8fc6cd25a1 /tidb/cdc/default/default/upstream/7365064247257976539 {"id":7365064247257976539,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6da06ee9-2891-4b79-a426-bc8fc6cd25a1 {"id":"6da06ee9-2891-4b79-a426-bc8fc6cd25a1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812663} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc734ace 6da06ee9-2891-4b79-a426-bc8fc6cd25a1 /tidb/cdc/default/default/upstream/7365064247257976539 {"id":7365064247257976539,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + generate_historic_files + local target_bucket=storage_test ++ date -d yesterday +%Y-%m-%d + yesterday=2024-05-03 ++ date -d '2 days ago' +%Y-%m-%d + day_before_yesterday=2024-05-02 + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_data_type 2024-05-03 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_data_type + local day=2024-05-03 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_charset 2024-05-02 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_charset + local day=2024-05-02 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test binary_columns 2024-05-02 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=binary_columns + local day=2024-05-02 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_data_type_dummy 2024-05-02 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_data_type_dummy + local day=2024-05-02 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_charset_dummy 2024-05-02 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_charset_dummy + local day=2024-05-02 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test binary_columns_dummy 2024-05-03 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=binary_columns_dummy + local day=2024-05-03 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test2 multi_data_type 2024-05-02 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test2 + local table=multi_data_type + local day=2024-05-02 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test2 multi_charset 2024-05-02 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test2 + local table=multi_charset + local day=2024-05-02 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test2 binary_columns 2024-05-03 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test2 + local table=binary_columns + local day=2024-05-03 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test_default test multi_data_type 2022-01-01 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test_default + local schema=test + local table=multi_data_type + local day=2022-01-01 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test_default test multi_charset 2022-01-02 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test_default + local schema=test + local table=multi_charset + local day=2022-01-02 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test_default test binary_columns 2022-01-03 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test_default + local schema=test + local table=binary_columns + local day=2022-01-03 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/meta/CDC.index + SINK_URI_DEFAULT='file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s' + run_cdc_cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s' -c default-config-test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed-default.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_cleanup.cli.3876.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s' -c default-config-test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed-default.toml check diff successfully Starting Upstream TiDB... Create changefeed successfully! ID: default-config-test Info: {"upstream_id":7365064247257976539,"namespace":"default","id":"default-config-test","sink_uri":"file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s","create_time":"2024-05-04T16:51:06.30152737+08:00","start_ts":449527851574362115,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527851574362115,"checkpoint_ts":449527851574362115,"checkpoint_time":"2024-05-04 16:51:06.223"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 1-th time... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... [Sat May 4 16:51:07 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + etcd_info_msg='etcd info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.52125214.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b32e7d00013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:4276, start at 2024-05-04 16:51:04.338789044 +0800 CST m=+5.241706685 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:04.347 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:04.308 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:04.308 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b32e7a00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:4361, start at 2024-05-04 16:51:04.330842844 +0800 CST m=+5.162969924 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:04.337 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:04.346 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:04.346 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table row_format.finish_mark not exists for 18-th check, retry later wait process cdc.test exit for 3-th time... + set +x + sleep 20 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:51:08 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 16:51:09 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.56115613.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 19-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:10 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/242d6ea4-964b-4a77-8a1d-c10168c32da9 {"id":"242d6ea4-964b-4a77-8a1d-c10168c32da9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812667} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc8170d4 242d6ea4-964b-4a77-8a1d-c10168c32da9 /tidb/cdc/default/default/upstream/7365064252434452066 {"id":7365064252434452066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/242d6ea4-964b-4a77-8a1d-c10168c32da9 {"id":"242d6ea4-964b-4a77-8a1d-c10168c32da9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812667} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc8170d4 242d6ea4-964b-4a77-8a1d-c10168c32da9 /tidb/cdc/default/default/upstream/7365064252434452066 {"id":7365064252434452066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/242d6ea4-964b-4a77-8a1d-c10168c32da9 {"id":"242d6ea4-964b-4a77-8a1d-c10168c32da9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812667} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc8170d4 242d6ea4-964b-4a77-8a1d-c10168c32da9 /tidb/cdc/default/default/upstream/7365064252434452066 {"id":7365064252434452066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5268.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json&enable-tidb-extension=true' -c=ddl-manager ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365064252434452066,"namespace":"default","id":"ddl-manager","sink_uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:51:10.989443395+08:00","start_ts":449527852786515974,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527852786515974,"checkpoint_ts":449527852786515974,"checkpoint_time":"2024-05-04 16:51:10.847"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 26.09 secs (142965404 bytes/sec) [Pipeline] { [Pipeline] cache table row_format.finish_mark not exists for 20-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + set +x + workdir=/tmp/tidb_cdc_test/ddl_manager + sink_uri='file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_manager ++ date + echo '[Sat May 4 16:51:12 CST 2024] <<<<<< START storage consumer in ddl_manager case >>>>>>' [Sat May 4 16:51:12 CST 2024] <<<<<< START storage consumer in ddl_manager case >>>>>> + cd /tmp/tidb_cdc_test/ddl_manager + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_manager + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_manager/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:12 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a80d1b0-6365-45e8-bfc7-55c2caa0fa26 {"id":"1a80d1b0-6365-45e8-bfc7-55c2caa0fa26","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812669} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc8dcacb 1a80d1b0-6365-45e8-bfc7-55c2caa0fa26 /tidb/cdc/default/default/upstream/7365064279920763701 {"id":7365064279920763701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a80d1b0-6365-45e8-bfc7-55c2caa0fa26 {"id":"1a80d1b0-6365-45e8-bfc7-55c2caa0fa26","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812669} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc8dcacb 1a80d1b0-6365-45e8-bfc7-55c2caa0fa26 /tidb/cdc/default/default/upstream/7365064279920763701 {"id":7365064279920763701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1a80d1b0-6365-45e8-bfc7-55c2caa0fa26 {"id":"1a80d1b0-6365-45e8-bfc7-55c2caa0fa26","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812669} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc8dcacb 1a80d1b0-6365-45e8-bfc7-55c2caa0fa26 /tidb/cdc/default/default/upstream/7365064279920763701 {"id":7365064279920763701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/changefeed_reconstruct + sink_uri='file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-31551?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_reconstruct ++ date + echo '[Sat May 4 16:51:12 CST 2024] <<<<<< START storage consumer in changefeed_reconstruct case >>>>>>' [Sat May 4 16:51:12 CST 2024] <<<<<< START storage consumer in changefeed_reconstruct case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_reconstruct + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_reconstruct + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-31551?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b335a400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr, pid:4701, start at 2024-05-04 16:51:11.67243402 +0800 CST m=+5.248000449 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:11.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:11.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:11.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "mysql.host"="127.0.0.1" "recordcount"="50" "readproportion"="0" "mysql.port"="4000" "mysql.user"="root" "updateproportion"="0" "requestdistribution"="uniform" "insertproportion"="0" "threadcount"="4" "mysql.db"="changefeed_reconstruct" "dotransactions"="false" "workload"="core" "readallfields"="true" "operationcount"="0" "scanproportion"="0" ********************************************** Run finished, takes 17.47993ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3590.2, Avg(us): 1366, Min(us): 924, Max(us): 4095, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later table row_format.finish_mark not exists for 21-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b335a400018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr, pid:4701, start at 2024-05-04 16:51:11.67243402 +0800 CST m=+5.248000449 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:11.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:11.682 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:11.682 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b335bf80010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-jjg-nl8qr, pid:4784, start at 2024-05-04 16:51:11.757272386 +0800 CST m=+5.253910312 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:11.763 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:11.742 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:11.742 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_reconstruct.usertable not exists for 2-th check, retry later table row_format.finish_mark not exists for 22-th check, retry later wait process 5217 exit for 1-th time... wait process 5217 exit for 2-th time... [Sat May 4 16:51:16 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.60906092.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_reconstruct.usertable not exists for 3-th check, retry later wait process 5217 exit for 3-th time... table row_format.finish_mark not exists for 23-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... wait process 5217 exit for 4-th time... wait process 5217 exit for 5-th time... wait process 5217 exit for 6-th time... table changefeed_reconstruct.usertable not exists for 4-th check, retry later wait process 5217 exit for 7-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:19 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/210d4183-09fa-46cc-a82e-d151c2a0b670 {"id":"210d4183-09fa-46cc-a82e-d151c2a0b670","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ccb42ccd 210d4183-09fa-46cc-a82e-d151c2a0b670 /tidb/cdc/default/default/upstream/7365064310128514813 {"id":7365064310128514813,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/210d4183-09fa-46cc-a82e-d151c2a0b670 {"id":"210d4183-09fa-46cc-a82e-d151c2a0b670","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ccb42ccd 210d4183-09fa-46cc-a82e-d151c2a0b670 /tidb/cdc/default/default/upstream/7365064310128514813 {"id":7365064310128514813,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/210d4183-09fa-46cc-a82e-d151c2a0b670 {"id":"210d4183-09fa-46cc-a82e-d151c2a0b670","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812677} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ccb42ccd 210d4183-09fa-46cc-a82e-d151c2a0b670 /tidb/cdc/default/default/upstream/7365064310128514813 {"id":7365064310128514813,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/changefeed_pause_resume + sink_uri='file:///tmp/tidb_cdc_test/changefeed_pause_resume/storage_test/ticdc-changefeed-pause-resume-27447?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_pause_resume ++ date + echo '[Sat May 4 16:51:20 CST 2024] <<<<<< START storage consumer in changefeed_pause_resume case >>>>>>' [Sat May 4 16:51:20 CST 2024] <<<<<< START storage consumer in changefeed_pause_resume case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_pause_resume + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_pause_resume + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_pause_resume/storage_test/ticdc-changefeed-pause-resume-27447?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' wait process 5217 exit for 8-th time... table row_format.finish_mark not exists for 24-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5217) - No such process wait process 5217 exit for 9-th time... process 5217 already exit [Sat May 4 16:51:20 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.53915393.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_pause_resume.t1 not exists for 1-th check, retry later table changefeed_reconstruct.usertable not exists for 5-th check, retry later table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:51:19 CST 2024] <<<<<< run test case sink_retry success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table row_format.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table changefeed_pause_resume.t1 not exists for 2-th check, retry later table changefeed_reconstruct.usertable exists check diff successfully wait process cdc.test exit for 3-th time... wait process 5616 exit for 1-th time... Verifying downstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:51:23 CST 2024] <<<<<< run test case row_format success! >>>>>> wait process 5616 exit for 2-th time... table changefeed_pause_resume.t1 not exists for 3-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5616) - No such process wait process 5616 exit for 3-th time... process 5616 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sat May 4 16:51:24 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.59205922.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:51:24 CST 2024] <<<<<< run test case cdc success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 13.58 secs (274620615 bytes/sec) [Pipeline] { < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365064252434452066 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:51:10.989443395 +0800 CST StartTs:449527852786515974 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037eda70 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527852812468231} {CheckpointTs:449527853219053595 MinTableBarrierTs:449527853219053595 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/856f735c-4fd8-4f30-adf8-987c405ca680 {"id":"856f735c-4fd8-4f30-adf8-987c405ca680","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812680} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc817365 856f735c-4fd8-4f30-adf8-987c405ca680 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365064252434452066,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:10.989443395+08:00","start-ts":449527852786515974,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527852812468231} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449527853245005829,"min-table-barrier-ts":449527853245005829,"admin-job-type":0} /tidb/cdc/default/default/task/position/856f735c-4fd8-4f30-adf8-987c405ca680/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064252434452066 {"id":7365064252434452066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365064252434452066 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:51:10.989443395 +0800 CST StartTs:449527852786515974 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037eda70 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527852812468231} {CheckpointTs:449527853219053595 MinTableBarrierTs:449527853219053595 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/856f735c-4fd8-4f30-adf8-987c405ca680 {"id":"856f735c-4fd8-4f30-adf8-987c405ca680","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812680} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc817365 856f735c-4fd8-4f30-adf8-987c405ca680 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365064252434452066,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:10.989443395+08:00","start-ts":449527852786515974,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527852812468231} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449527853245005829,"min-table-barrier-ts":449527853245005829,"admin-job-type":0} /tidb/cdc/default/default/task/position/856f735c-4fd8-4f30-adf8-987c405ca680/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064252434452066 + grep -q 'failed to get info:' {"id":7365064252434452066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365064252434452066 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:51:10.989443395 +0800 CST StartTs:449527852786515974 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037eda70 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527852812468231} {CheckpointTs:449527853219053595 MinTableBarrierTs:449527853219053595 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/856f735c-4fd8-4f30-adf8-987c405ca680 {"id":"856f735c-4fd8-4f30-adf8-987c405ca680","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812680} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc817365 856f735c-4fd8-4f30-adf8-987c405ca680 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365064252434452066,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:10.989443395+08:00","start-ts":449527852786515974,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527852812468231} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449527853245005829,"min-table-barrier-ts":449527853245005829,"admin-job-type":0} /tidb/cdc/default/default/task/position/856f735c-4fd8-4f30-adf8-987c405ca680/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064252434452066 {"id":7365064252434452066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 16:51:27 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.54595461.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [Pipeline] cache table changefeed_pause_resume.t1 not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:27 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/2ce3eac4-a739-402b-963a-a764908f2378 {UpstreamID:7365064279920763701 Namespace:default ID:2ce3eac4-a739-402b-963a-a764908f2378 SinkURI:file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-31551?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:51:12.712994506 +0800 CST StartTs:449527853248151556 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0019d3710 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527853274365953} {CheckpointTs:449527855764733953 MinTableBarrierTs:449527857075453953 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449527855764733953, checkpointTs: 449527855764733953, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df74b4c0-893f-49c0-ab63-9d387f9b0a89 {"id":"df74b4c0-893f-49c0-ab63-9d387f9b0a89","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812685} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc8dcb8d df74b4c0-893f-49c0-ab63-9d387f9b0a89 /tidb/cdc/default/default/changefeed/info/2ce3eac4-a739-402b-963a-a764908f2378 {"upstream-id":7365064279920763701,"namespace":"default","changefeed-id":"2ce3eac4-a739-402b-963a-a764908f2378","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-31551?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:12.712994506+08:00","start-ts":449527853248151556,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527853274365953} /tidb/cdc/default/default/changefeed/status/2ce3eac4-a739-402b-963a-a764908f2378 {"checkpoint-ts":449527855764733953,"min-table-barrier-ts":449527857075453953,"admin-job-type":0} /tidb/cdc/default/default/task/position/df74b4c0-893f-49c0-ab63-9d387f9b0a89/2ce3eac4-a739-402b-963a-a764908f2378 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064279920763701 {"id":7365064279920763701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2ce3eac4-a739-402b-963a-a764908f2378 {UpstreamID:7365064279920763701 Namespace:default ID:2ce3eac4-a739-402b-963a-a764908f2378 SinkURI:file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-31551?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:51:12.712994506 +0800 CST StartTs:449527853248151556 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0019d3710 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527853274365953} {CheckpointTs:449527855764733953 MinTableBarrierTs:449527857075453953 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449527855764733953, checkpointTs: 449527855764733953, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df74b4c0-893f-49c0-ab63-9d387f9b0a89 {"id":"df74b4c0-893f-49c0-ab63-9d387f9b0a89","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812685} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc8dcb8d df74b4c0-893f-49c0-ab63-9d387f9b0a89 /tidb/cdc/default/default/changefeed/info/2ce3eac4-a739-402b-963a-a764908f2378 {"upstream-id":7365064279920763701,"namespace":"default","changefeed-id":"2ce3eac4-a739-402b-963a-a764908f2378","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-31551?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:12.712994506+08:00","start-ts":449527853248151556,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527853274365953} /tidb/cdc/default/default/changefeed/status/2ce3eac4-a739-402b-963a-a764908f2378 {"checkpoint-ts":449527855764733953,"min-table-barrier-ts":449527857075453953,"admin-job-type":0} /tidb/cdc/default/default/task/position/df74b4c0-893f-49c0-ab63-9d387f9b0a89/2ce3eac4-a739-402b-963a-a764908f2378 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064279920763701 {"id":7365064279920763701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/2ce3eac4-a739-402b-963a-a764908f2378 {UpstreamID:7365064279920763701 Namespace:default ID:2ce3eac4-a739-402b-963a-a764908f2378 SinkURI:file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-31551?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:51:12.712994506 +0800 CST StartTs:449527853248151556 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0019d3710 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527853274365953} {CheckpointTs:449527855764733953 MinTableBarrierTs:449527857075453953 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449527855764733953, checkpointTs: 449527855764733953, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/df74b4c0-893f-49c0-ab63-9d387f9b0a89 {"id":"df74b4c0-893f-49c0-ab63-9d387f9b0a89","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812685} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc8dcb8d df74b4c0-893f-49c0-ab63-9d387f9b0a89 /tidb/cdc/default/default/changefeed/info/2ce3eac4-a739-402b-963a-a764908f2378 {"upstream-id":7365064279920763701,"namespace":"default","changefeed-id":"2ce3eac4-a739-402b-963a-a764908f2378","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-31551?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:12.712994506+08:00","start-ts":449527853248151556,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527853274365953} /tidb/cdc/default/default/changefeed/status/2ce3eac4-a739-402b-963a-a764908f2378 {"checkpoint-ts":449527855764733953,"min-table-barrier-ts":449527857075453953,"admin-job-type":0} /tidb/cdc/default/default/task/position/df74b4c0-893f-49c0-ab63-9d387f9b0a89/2ce3eac4-a739-402b-963a-a764908f2378 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":nul+ grep -q 'etcd info' l,"warning":null} /tidb/cdc/default/default/upstream/7365064279920763701 {"id":7365064279920763701,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:29 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365064252434452066 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:51:10.989443395 +0800 CST StartTs:449527852786515974 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037eda70 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527852812468231} {CheckpointTs:449527853245005829 MinTableBarrierTs:449527853245005829 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/856f735c-4fd8-4f30-adf8-987c405ca680 {"id":"856f735c-4fd8-4f30-adf8-987c405ca680","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812680} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc817365 856f735c-4fd8-4f30-adf8-987c405ca680 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365064252434452066,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:10.989443395+08:00","start-ts":449527852786515974,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527852812468231} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449527853245005829,"min-table-barrier-ts":449527853245005829,"admin-job-type":0} /tidb/cdc/default/default/task/position/856f735c-4fd8-4f30-adf8-987c405ca680/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064252434452066 {"id":7365064252434452066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365064252434452066 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:51:10.989443395 +0800 CST StartTs:449527852786515974 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037eda70 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527852812468231} {CheckpointTs:449527853245005829 MinTableBarrierTs:449527853245005829 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/856f735c-4fd8-4f30-adf8-987c405ca680 {"id":"856f735c-4fd8-4f30-adf8-987c405ca680","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812680} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc817365 856f735c-4fd8-4f30-adf8-987c405ca680 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365064252434452066,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:10.989443395+08:00","start-ts":449527852786515974,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527852812468231} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449527853245005829,"min-table-barrier-ts":449527853245005829,"admin-job-type":0} /tidb/cdc/default/default/task/position/856f735c-4fd8-4f30-adf8-987c405ca680/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064252434452066 {"id":7365064252434452066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365064252434452066 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:51:10.989443395 +0800 CST StartTs:449527852786515974 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0037eda70 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527852812468231} {CheckpointTs:449527853245005829 MinTableBarrierTs:449527853245005829 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/856f735c-4fd8-4f30-adf8-987c405ca680 {"id":"856f735c-4fd8-4f30-adf8-987c405ca680","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812680} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cc817365 856f735c-4fd8-4f30-adf8-987c405ca680 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365064252434452066,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-4564?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:10.989443395+08:00","start-ts":449527852786515974,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527852812468231} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449527853245005829,"min-table-barrier-ts":449527853245005829,"admin-job-type":0} /tidb/cdc/default/default/task/position/856f735c-4fd8-4f30-adf8-987c405ca680/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064252434452066 {"id":7365064252434452066,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later table changefeed_pause_resume.t1 not exists for 5-th check, retry later "id": "df74b4c0-893f-49c0-ab63-9d387f9b0a89", "cluster-id": "default" run task successfully + check_file_exists true + local all_should_exist=true + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/10.data ']' + SINK_URI='file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' + run_cdc_cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_cleanup.cli.3937.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml Create changefeed successfully! ID: 0db831dd-9664-4396-a8ce-36212db8ca9e Info: {"upstream_id":7365064247257976539,"namespace":"default","id":"0db831dd-9664-4396-a8ce-36212db8ca9e","sink_uri":"file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s","create_time":"2024-05-04T16:51:28.801780462+08:00","start_ts":449527857472602116,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"cloud_storage_config":{"file_expiration_days":1,"file_cleanup_cron_spec":"* * * * * *"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527857472602116,"checkpoint_ts":449527857472602116,"checkpoint_time":"2024-05-04 16:51:28.723"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + sleep 20 capture_id: df74b4c0-893f-49c0-ab63-9d387f9b0a89 check_processor_table_count http://127.0.0.1:2379 2ce3eac4-a739-402b-963a-a764908f2378 df74b4c0-893f-49c0-ab63-9d387f9b0a89 1 table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 exists run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check_processor_table_count http://127.0.0.1:2379 2ce3eac4-a739-402b-963a-a764908f2378 df74b4c0-893f-49c0-ab63-9d387f9b0a89 0 table ddl_manager.finish_mark not exists for 2-th check, retry later table count 1 does equal to expected count 0 run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 3-th check, retry later check_processor_table_count http://127.0.0.1:2379 2ce3eac4-a739-402b-963a-a764908f2378 df74b4c0-893f-49c0-ab63-9d387f9b0a89 0 run task successfully ***************** properties ***************** "mysql.user"="root" "mysql.port"="4000" "operationcount"="0" "workload"="core" "readallfields"="true" "updateproportion"="0" "readproportion"="0" "mysql.db"="changefeed_reconstruct" "mysql.host"="127.0.0.1" "recordcount"="50" "threadcount"="4" "scanproportion"="0" "requestdistribution"="uniform" "insertproportion"="0" "dotransactions"="false" ********************************************** Run finished, takes 17.894019ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3720.8, Avg(us): 1425, Min(us): 881, Max(us): 5019, 95th(us): 5000, 99th(us): 6000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b34a2440007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7, pid:3884, start at 2024-05-04 16:51:32.632735834 +0800 CST m=+5.139011646 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:32.641 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:32.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:32.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b34a2440007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7, pid:3884, start at 2024-05-04 16:51:32.632735834 +0800 CST m=+5.139011646 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:32.641 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:32.625 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:32.625 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b34a3a80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-7sv-ndvz7, pid:3968, start at 2024-05-04 16:51:32.745030685 +0800 CST m=+5.198413264 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:32.750 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:32.714 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:32.714 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_reconstruct.usertable not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5282.out cli tso query --pd=http://127.0.0.1:2379 table changefeed_reconstruct.usertable not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449527860055506945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527860055506945 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 16:51:40 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.53335335.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_reconstruct.usertable not exists for 4-th check, retry later check diff failed 4-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b35199c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj, pid:4429, start at 2024-05-04 16:51:40.276040225 +0800 CST m=+5.174045489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:40.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:40.263 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:40.263 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b35199c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj, pid:4429, start at 2024-05-04 16:51:40.276040225 +0800 CST m=+5.174045489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:40.282 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:40.263 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:40.263 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b351bd0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-80l-0bpmj, pid:4505, start at 2024-05-04 16:51:40.41760042 +0800 CST m=+5.259155561 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:40.423 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:40.404 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:40.404 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 7-th check, retry later table changefeed_reconstruct.usertable not exists for 5-th check, retry later check diff failed 5-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.5755.out cli tso query --pd=http://127.0.0.1:2379 table ddl_manager.finish_mark not exists for 8-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:43 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dacdc2c1-b2f8-4a30-bc66-a769b87a1a04 {"id":"dacdc2c1-b2f8-4a30-bc66-a769b87a1a04","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812700} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ccfe5bf5 dacdc2c1-b2f8-4a30-bc66-a769b87a1a04 /tidb/cdc/default/default/upstream/7365064391017494965 {"id":7365064391017494965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dacdc2c1-b2f8-4a30-bc66-a769b87a1a04 {"id":"dacdc2c1-b2f8-4a30-bc66-a769b87a1a04","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812700} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ccfe5bf5 dacdc2c1-b2f8-4a30-bc66-a769b87a1a04 /tidb/cdc/default/default/upstream/7365064391017494965 {"id":7365064391017494965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dacdc2c1-b2f8-4a30-bc66-a769b87a1a04 {"id":"dacdc2c1-b2f8-4a30-bc66-a769b87a1a04","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812700} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ccfe5bf5 dacdc2c1-b2f8-4a30-bc66-a769b87a1a04 /tidb/cdc/default/default/upstream/7365064391017494965 {"id":7365064391017494965,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5390.out cli changefeed create --start-ts=449527860055506945 '--sink-uri=file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-29776?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 9c189495-52c4-4e53-a917-cc082125bc03 Info: {"upstream_id":7365064391017494965,"namespace":"default","id":"9c189495-52c4-4e53-a917-cc082125bc03","sink_uri":"file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-29776?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:51:43.799755257+08:00","start_ts":449527860055506945,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527860055506945,"checkpoint_ts":449527860055506945,"checkpoint_time":"2024-05-04 16:51:38.576"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table changefeed_reconstruct.usertable exists check diff failed 6-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully wait process cdc.test exit for 1-th time... + set +x + workdir=/tmp/tidb_cdc_test/ddl_puller_lag + sink_uri='file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-29776?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_puller_lag ++ date + echo '[Sat May 4 16:51:45 CST 2024] <<<<<< START storage consumer in ddl_puller_lag case >>>>>>' [Sat May 4 16:51:45 CST 2024] <<<<<< START storage consumer in ddl_puller_lag case >>>>>> + cd /tmp/tidb_cdc_test/ddl_puller_lag + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_puller_lag + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-29776?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + set +x table ddl_manager.finish_mark not exists for 9-th check, retry later + tso='449527861402140673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527861402140673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x wait process cdc.test exit for 2-th time... [Sat May 4 16:51:45 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.58075809.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:51:45 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 7-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 10-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 8-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:48 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73a1c777-fd66-455a-be11-26f97d31d7d4 {"id":"73a1c777-fd66-455a-be11-26f97d31d7d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812705} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd223ff6 73a1c777-fd66-455a-be11-26f97d31d7d4 /tidb/cdc/default/default/upstream/7365064431968088534 {"id":7365064431968088534,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73a1c777-fd66-455a-be11-26f97d31d7d4 {"id":"73a1c777-fd66-455a-be11-26f97d31d7d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812705} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd223ff6 73a1c777-fd66-455a-be11-26f97d31d7d4 /tidb/cdc/default/default/upstream/7365064431968088534 {"id":7365064431968088534,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/73a1c777-fd66-455a-be11-26f97d31d7d4 {"id":"73a1c777-fd66-455a-be11-26f97d31d7d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812705} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd223ff6 73a1c777-fd66-455a-be11-26f97d31d7d4 /tidb/cdc/default/default/upstream/7365064431968088534 {"id":7365064431968088534,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.5864.out cli changefeed create --start-ts=449527861402140673 '--sink-uri=file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-14408?protocol=canal-json&enable-tidb-extension=true' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b356ec80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs, pid:3939, start at 2024-05-04 16:51:45.739421363 +0800 CST m=+5.334829463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:45.745 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:45.714 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:45.714 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b356ec80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs, pid:3939, start at 2024-05-04 16:51:45.739421363 +0800 CST m=+5.334829463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:45.745 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:45.714 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:45.714 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3570500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs, pid:4006, start at 2024-05-04 16:51:45.858823505 +0800 CST m=+5.398546361 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:45.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:45.862 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:45.862 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 11-th check, retry later Create changefeed successfully! ID: 87439a40-b3b5-4982-826d-e11ab5aa684f Info: {"upstream_id":7365064431968088534,"namespace":"default","id":"87439a40-b3b5-4982-826d-e11ab5aa684f","sink_uri":"file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-14408?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:51:49.037312116+08:00","start_ts":449527861402140673,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527861402140673,"checkpoint_ts":449527861402140673,"checkpoint_time":"2024-05-04 16:51:43.713"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5255.out cli tso query --pd=http://127.0.0.1:2379 + set +x + workdir=/tmp/tidb_cdc_test/split_region + sink_uri='file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-14408?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/split_region ++ date + echo '[Sat May 4 16:51:50 CST 2024] <<<<<< START storage consumer in split_region case >>>>>>' [Sat May 4 16:51:50 CST 2024] <<<<<< START storage consumer in split_region case >>>>>> + cd /tmp/tidb_cdc_test/split_region + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/split_region + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/split_region/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-14408?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table split_region.test1 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 12-th check, retry later + check_file_exists false + local all_should_exist=false + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-03/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-03/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-03/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-02/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-02/10.data ']' + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/data/schema.sql 127.0.0.1 4000 + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/data/data.sql 127.0.0.1 4000 + run_storage_consumer /tmp/tidb_cdc_test/storage_cleanup 'file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml '' + workdir=/tmp/tidb_cdc_test/storage_cleanup + sink_uri='file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Sat May 4 16:51:50 CST 2024] <<<<<< START storage consumer in storage_cleanup case >>>>>>' [Sat May 4 16:51:50 CST 2024] <<<<<< START storage consumer in storage_cleanup case >>>>>> + cd /tmp/tidb_cdc_test/storage_cleanup + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/storage_cleanup + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/storage_cleanup/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml + sleep 8 check diff failed 1-th time, retry later + set +x + tso='449527863329423361 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527863329423361 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table split_region.test1 not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 13-th check, retry later ***************** properties ***************** "dotransactions"="false" "operationcount"="0" "scanproportion"="0" "mysql.port"="4000" "requestdistribution"="uniform" "mysql.db"="changefeed_error" "threadcount"="4" "mysql.user"="root" "mysql.host"="127.0.0.1" "insertproportion"="0" "recordcount"="20" "workload"="core" "readallfields"="true" "readproportion"="0" "updateproportion"="0" ********************************************** Run finished, takes 9.63072ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3771.2, Avg(us): 1813, Min(us): 963, Max(us): 4309, 95th(us): 5000, 99th(us): 5000 [Sat May 4 16:51:53 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.53145316.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 24.27 secs (153671456 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table split_region.test1 not exists for 3-th check, retry later check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b35e67c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw, pid:4006, start at 2024-05-04 16:51:53.420634869 +0800 CST m=+5.341068798 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:53.427 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:53.426 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:53.426 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b35e67c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw, pid:4006, start at 2024-05-04 16:51:53.420634869 +0800 CST m=+5.341068798 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:53.427 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:53.426 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:53.426 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b35e98c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw, pid:4085, start at 2024-05-04 16:51:53.596130832 +0800 CST m=+5.465469835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:53:53.603 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:51:53.571 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:41:53.571 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table split_region.test1 not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 14-th check, retry later check diff failed 3-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:51:56 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 {"id":"75362273-63d0-42c6-876e-9cd13b7464d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812714} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f27f2 75362273-63d0-42c6-876e-9cd13b7464d4 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 {"id":"75362273-63d0-42c6-876e-9cd13b7464d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812714} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f27f2 75362273-63d0-42c6-876e-9cd13b7464d4 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 {"id":"75362273-63d0-42c6-876e-9cd13b7464d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812714} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f27f2 75362273-63d0-42c6-876e-9cd13b7464d4 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5372.out cli changefeed create --start-ts=449527863329423361 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365064454781274033,"namespace":"default","id":"changefeed-error","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:51:57.204918447+08:00","start_ts":449527863329423361,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527863329423361,"checkpoint_ts":449527863329423361,"checkpoint_time":"2024-05-04 16:51:51.065"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff failed 4-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5345.out cli tso query --pd=http://127.0.0.1:2379 + check_sync_diff /tmp/tidb_cdc_test/storage_cleanup /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/diff_config.toml 100 check diff failed 1-th time, retry later table split_region.test1 not exists for 5-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/changefeed_error + sink_uri='file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_error ++ date + echo '[Sat May 4 16:51:58 CST 2024] <<<<<< START storage consumer in changefeed_error case >>>>>>' [Sat May 4 16:51:58 CST 2024] <<<<<< START storage consumer in changefeed_error case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_error + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_error + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_error/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449527863329423361, "checkpoint_time": "2024-05-04 16:51:51.065", "error": { "time": "2024-05-04T16:51:57.277923688+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449527863329423061 is earlier than or equal to GC safepoint at 449527863329423361" } }' + echo '{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449527863329423361, "checkpoint_time": "2024-05-04 16:51:51.065", "error": { "time": "2024-05-04T16:51:57.277923688+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449527863329423061 is earlier than or equal to GC safepoint at 449527863329423361" } }' { "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449527863329423361, "checkpoint_time": "2024-05-04 16:51:51.065", "error": { "time": "2024-05-04T16:51:57.277923688+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449527863329423061 is earlier than or equal to GC safepoint at 449527863329423361" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449527863329423361, '"checkpoint_time":' '"2024-05-04' '16:51:51.065",' '"error":' '{' '"time":' '"2024-05-04T16:51:57.277923688+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449527863329423061 is earlier than or equal to GC safepoint at '449527863329423361"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449527863329423361, '"checkpoint_time":' '"2024-05-04' '16:51:51.065",' '"error":' '{' '"time":' '"2024-05-04T16:51:57.277923688+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449527863329423061 is earlier than or equal to GC safepoint at '449527863329423361"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449527863329423061 is earlier than or equal to GC safepoint at 449527863329423361' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449527863329423061 is earlier than or equal to GC safepoint at 449527863329423361 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5456.out cli changefeed resume -c changefeed-error table ddl_manager.finish_mark not exists for 15-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + set +x + tso='449527865337446401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527865337446401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 16:52:00 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.53815383.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 2-th time, retry later table split_region.test1 not exists for 6-th check, retry later + set +x table changefeed_error.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 16-th check, retry later check diff failed 5-th time, retry later table split_region.test1 exists table split_region.test2 exists check diff successfully table ddl_manager.finish_mark not exists for 17-th check, retry later check diff failed 6-th time, retry later check diff successfully + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Sat May 4 16:52:02 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Sat May 4 16:52:02 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:03 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1942bb3a-45e6-4710-ba9a-d64de7dc4f88 {"id":"1942bb3a-45e6-4710-ba9a-d64de7dc4f88","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812720} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd52ecd5 1942bb3a-45e6-4710-ba9a-d64de7dc4f88 /tidb/cdc/default/default/upstream/7365064485067593134 {"id":7365064485067593134,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1942bb3a-45e6-4710-ba9a-d64de7dc4f88 {"id":"1942bb3a-45e6-4710-ba9a-d64de7dc4f88","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812720} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd52ecd5 1942bb3a-45e6-4710-ba9a-d64de7dc4f88 /tidb/cdc/default/default/upstream/7365064485067593134 {"id":7365064485067593134,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1942bb3a-45e6-4710-ba9a-d64de7dc4f88 {"id":"1942bb3a-45e6-4710-ba9a-d64de7dc4f88","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812720} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd52ecd5 1942bb3a-45e6-4710-ba9a-d64de7dc4f88 /tidb/cdc/default/default/upstream/7365064485067593134 {"id":7365064485067593134,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5436.out cli changefeed create --start-ts=449527865337446401 '--sink-uri=file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-21664?protocol=canal-json&enable-tidb-extension=true' table changefeed_error.usertable not exists for 2-th check, retry later Create changefeed successfully! ID: 3cb81aa8-c688-4f9b-b1dd-c0d57bdbce0e Info: {"upstream_id":7365064485067593134,"namespace":"default","id":"3cb81aa8-c688-4f9b-b1dd-c0d57bdbce0e","sink_uri":"file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-21664?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:03.743136602+08:00","start_ts":449527865337446401,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527865337446401,"checkpoint_ts":449527865337446401,"checkpoint_time":"2024-05-04 16:51:58.725"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 18-th check, retry later check diff failed 7-th time, retry later + set +x + workdir=/tmp/tidb_cdc_test/drop_many_tables + sink_uri='file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-21664?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/drop_many_tables ++ date + echo '[Sat May 4 16:52:05 CST 2024] <<<<<< START storage consumer in drop_many_tables case >>>>>>' [Sat May 4 16:52:05 CST 2024] <<<<<< START storage consumer in drop_many_tables case >>>>>> + cd /tmp/tidb_cdc_test/drop_many_tables + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/drop_many_tables + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-21664?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table changefeed_error.usertable not exists for 3-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc) 3730343424 bytes in 9.15 secs (407499222 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh Starting Upstream TiDB... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=973628ad-92d6-4ab8-8b3c-b9cd5e1cd661 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-n0x62 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-n0x62 pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh table changefeed_error.usertable not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 19-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=5b9d580b-97b3-40f7-a2be-27319c3c218f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-pb1r6 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0 pingcap_tiflow_pull_cdc_integration_storage_test_1781-pb1r6 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:52:07 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> check diff failed 8-th time, retry later [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=5082bd10-b11e-41c4-a69b-515811f56252 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-rlkxk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-rlkxk pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: storage... <<================= + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=b9e22a43-56b0-4385-8b7c-0f2366561400 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-bzq0f GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l pingcap_tiflow_pull_cdc_integration_storage_test_1781-bzq0f GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:52:08 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=b514073a-2b45-4aa1-b079-ee3d52f491fd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-5v88p GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-5v88p pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:52:08 CST 2024] <<<<<< skip test case clustered_index for storage! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=0ee8e260-484a-44b1-ad91-a67f61076231 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-kh4bj GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1781-kh4bj pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: storage... <<================= [Sat May 4 16:52:08 CST 2024] <<<<<< run test case default_value success! >>>>>> [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=9d73541f-fc27-4047-ad17-a3f709549dcd BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-nbjsn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g pingcap_tiflow_pull_cdc_integration_storage_test_1781-nbjsn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] // container [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=c43fbabd-2572-4710-9a30-065b81d301af JENKINS_NODE_COOKIE=ccd807f3-655e-4978-9899-08105df9779f BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786675747499282437","prowjobid":"c43fbabd-2572-4710-9a30-065b81d301af","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/c950cce3a9b105fd95bb2c788e1ab69ec32e0668","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1781 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786675747499282437 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1781/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1781-29bjp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9 pingcap_tiflow_pull_cdc_integration_storage_test_1781-29bjp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1781 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table changefeed_error.usertable exists check diff successfully table ddl_manager.finish_mark not exists for 20-th check, retry later check diff failed 9-th time, retry later ***************** properties ***************** "readallfields"="true" "mysql.port"="4000" "scanproportion"="0" "operationcount"="0" "workload"="core" "mysql.db"="changefeed_error" "updateproportion"="0" "readproportion"="0" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.user"="root" "recordcount"="20" "threadcount"="4" "insertproportion"="0" ********************************************** Run finished, takes 4.916517ms INSERT - Takes(s): 0.0, Count: 20, OPS: 5543.5, Avg(us): 853, Min(us): 465, Max(us): 2124, 95th(us): 3000, 99th(us): 3000 check diff successfully {"id":"75362273-63d0-42c6-876e-9cd13b7464d4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812714} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 + [[ /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later table drop_tables.c not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 21-th check, retry later check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table drop_tables.c not exists for 2-th check, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 + [[ /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 22-th check, retry later check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3709b00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:7308, start at 2024-05-04 16:52:12.06007337 +0800 CST m=+5.311168011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:12.067 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:12.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:12.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc already exists) table drop_tables.c not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 23-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3709b00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:7308, start at 2024-05-04 16:52:12.06007337 +0800 CST m=+5.311168011 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:12.067 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:12.062 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:12.062 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b370b28000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:7388, start at 2024-05-04 16:52:12.11485645 +0800 CST m=+5.313890756 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:12.121 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:12.106 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:12.106 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 2-th time, retry later table drop_tables.c not exists for 4-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 + [[ /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/75362273-63d0-42c6-876e-9cd13b7464d4 + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later The 1 times to try to start tidb cluster... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 24-th check, retry later Verifying downstream PD is started... check diff failed 3-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8639.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 1-th time, retry later table drop_tables.c not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449527870236393473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527870236393473 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 25-th check, retry later ***************** properties ***************** "readallfields"="true" "mysql.host"="127.0.0.1" "recordcount"="10" "operationcount"="0" "requestdistribution"="uniform" "dotransactions"="false" "insertproportion"="0" "mysql.user"="root" "updateproportion"="0" "threadcount"="2" "mysql.db"="multi_capture_1" "workload"="core" "scanproportion"="0" "readproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 9.287303ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1888.9, Avg(us): 1774, Min(us): 1115, Max(us): 3968, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "readproportion"="0" "mysql.db"="multi_capture_2" "recordcount"="10" "dotransactions"="false" "scanproportion"="0" "readallfields"="true" "mysql.user"="root" "mysql.host"="127.0.0.1" "operationcount"="0" "requestdistribution"="uniform" "updateproportion"="0" "threadcount"="2" "workload"="core" "insertproportion"="0" "mysql.port"="4000" ********************************************** Run finished, takes 8.824369ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2003.9, Avg(us): 1706, Min(us): 1098, Max(us): 3744, 95th(us): 4000, 99th(us): 4000 check diff failed 4-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ***************** properties ***************** "operationcount"="0" "workload"="core" "mysql.port"="4000" "requestdistribution"="uniform" "recordcount"="10" "threadcount"="2" "mysql.db"="multi_capture_3" "readallfields"="true" "insertproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "readproportion"="0" "updateproportion"="0" "mysql.user"="root" "dotransactions"="false" ********************************************** Run finished, takes 10.22809ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1698.7, Avg(us): 1967, Min(us): 1219, Max(us): 4296, 95th(us): 5000, 99th(us): 5000 table drop_tables.c not exists for 6-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ***************** properties ***************** "insertproportion"="0" "requestdistribution"="uniform" "mysql.user"="root" "workload"="core" "mysql.db"="multi_capture_4" "scanproportion"="0" "readallfields"="true" "recordcount"="10" "mysql.port"="4000" "threadcount"="2" "mysql.host"="127.0.0.1" "updateproportion"="0" "dotransactions"="false" "readproportion"="0" "operationcount"="0" ********************************************** Run finished, takes 9.368388ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1925.3, Avg(us): 1802, Min(us): 1107, Max(us): 4153, 95th(us): 5000, 99th(us): 5000 [Sat May 4 16:52:19 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.87648766.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... check diff failed 2-th time, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 26-th check, retry later table drop_tables.c not exists for 7-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 5-th time, retry later check diff successfully Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3799100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27, pid:1305, start at 2024-05-04 16:52:21.206362664 +0800 CST m=+5.226180801 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:21.215 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:21.188 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:21.188 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 16:52:21 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.56855687.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // withEnv VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37a5c0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9, pid:1229, start at 2024-05-04 16:52:22.011107175 +0800 CST m=+5.466955780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:22.017 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:22.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:22.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37a7ac000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g, pid:1289, start at 2024-05-04 16:52:22.134200457 +0800 CST m=+5.368404759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:22.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:22.123 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:22.123 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 27-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:22 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a90ac6a-69c4-4f86-9102-7f5afc873773 {"id":"9a90ac6a-69c4-4f86-9102-7f5afc873773","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812740} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96123a 9a90ac6a-69c4-4f86-9102-7f5afc873773 /tidb/cdc/default/default/upstream/7365064565591594020 {"id":7365064565591594020,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a90ac6a-69c4-4f86-9102-7f5afc873773 {"id":"9a90ac6a-69c4-4f86-9102-7f5afc873773","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812740} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96123a 9a90ac6a-69c4-4f86-9102-7f5afc873773 /tidb/cdc/default/default/upstream/7365064565591594020 {"id":7365064565591594020,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a90ac6a-69c4-4f86-9102-7f5afc873773 {"id":"9a90ac6a-69c4-4f86-9102-7f5afc873773","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812740} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96123a 9a90ac6a-69c4-4f86-9102-7f5afc873773 /tidb/cdc/default/default/upstream/7365064565591594020 {"id":7365064565591594020,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 16:52:22 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.88218823.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } [Pipeline] // cache wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:52:23 CST 2024] <<<<<< run test case split_region success! >>>>>> check diff failed 6-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table drop_tables.c not exists for 8-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37c55c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5, pid:1278, start at 2024-05-04 16:52:24.024666805 +0800 CST m=+6.040729757 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:24.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:24.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:24.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3799100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27, pid:1305, start at 2024-05-04 16:52:21.206362664 +0800 CST m=+5.226180801 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:21.215 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:21.188 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:21.188 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37a3f40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27, pid:1387, start at 2024-05-04 16:52:21.916468486 +0800 CST m=+5.881898704 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:21.925 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:21.885 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:21.885 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:24 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37a5c0000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9, pid:1229, start at 2024-05-04 16:52:22.011107175 +0800 CST m=+5.466955780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:22.017 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:22.000 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:22.000 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37a73c000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-29b-gbxc9, pid:1311, start at 2024-05-04 16:52:22.103433685 +0800 CST m=+5.504903983 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:22.113 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:22.095 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:22.095 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83aaf3af-e318-4f9d-987f-9abfe0586af6 {"id":"83aaf3af-e318-4f9d-987f-9abfe0586af6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f28fd 83aaf3af-e318-4f9d-987f-9abfe0586af6 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365064454781274033,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:57.204918447+08:00","start-ts":449527863329423361,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T16:52:22.225958245+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527871495208962} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449527867890466820,"min-table-barrier-ts":449527867890466820,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83aaf3af-e318-4f9d-987f-9abfe0586af6 {"id":"83aaf3af-e318-4f9d-987f-9abfe0586af6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f28fd 83aaf3af-e318-4f9d-987f-9abfe0586af6 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365064454781274033,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:57.204918447+08:00","start-ts":449527863329423361,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T16:52:22.225958245+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527871495208962} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449527867890466820,"min-table-barrier-ts":449527867890466820,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83aaf3af-e318-4f9d-987f-9abfe0586af6 {"id":"83aaf3af-e318-4f9d-987f-9abfe0586af6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812741} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f28fd 83aaf3af-e318-4f9d-987f-9abfe0586af6 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365064454781274033,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:51:57.204918447+08:00","start-ts":449527863329423361,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-04T16:52:22.225958245+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527871495208962} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449527867890466820,"min-table-barrier-ts":449527867890466820,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449527867890466820, "checkpoint_time": "2024-05-04 16:52:08.464", "error": { "time": "2024-05-04T16:52:22.225958245+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449527867890466820, "checkpoint_time": "2024-05-04 16:52:08.464", "error": { "time": "2024-05-04T16:52:22.225958245+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449527867890466820, "checkpoint_time": "2024-05-04 16:52:08.464", "error": { "time": "2024-05-04T16:52:22.225958245+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449527867890466820, '"checkpoint_time":' '"2024-05-04' '16:52:08.464",' '"error":' '{' '"time":' '"2024-05-04T16:52:22.225958245+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449527867890466820, '"checkpoint_time":' '"2024-05-04' '16:52:08.464",' '"error":' '{' '"time":' '"2024-05-04T16:52:22.225958245+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message table ddl_manager.finish_mark not exists for 28-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5767.out cli changefeed remove -c changefeed-error [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37a7ac000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g, pid:1289, start at 2024-05-04 16:52:22.134200457 +0800 CST m=+5.368404759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:22.141 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:22.123 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:22.123 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37a798000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g, pid:1381, start at 2024-05-04 16:52:22.13137186 +0800 CST m=+5.265390285 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:22.139 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:22.118 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:22.118 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 7-th time, retry later [Pipeline] // node [Pipeline] } table drop_tables.c not exists for 9-th check, retry later Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449527867890466820 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // dir + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:25 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1457ffe1-225c-4165-81dd-3b2080e0d0a4 {"id":"1457ffe1-225c-4165-81dd-3b2080e0d0a4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812743} /tidb/cdc/default/__cdc_meta__/capture/9a90ac6a-69c4-4f86-9102-7f5afc873773 {"id":"9a90ac6a-69c4-4f86-9102-7f5afc873773","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812740} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96123a 9a90ac6a-69c4-4f86-9102-7f5afc873773 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96126c 1457ffe1-225c-4165-81dd-3b2080e0d0a4 /tidb/cdc/default/default/upstream/7365064565591594020 {"id":7365064565591594020,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1457ffe1-225c-4165-81dd-3b2080e0d0a4 {"id":"1457ffe1-225c-4165-81dd-3b2080e0d0a4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812743} /tidb/cdc/default/__cdc_meta__/capture/9a90ac6a-69c4-4f86-9102-7f5afc873773 {"id":"9a90ac6a-69c4-4f86-9102-7f5afc873773","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812740} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96123a 9a90ac6a-69c4-4f86-9102-7f5afc873773 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96126c 1457ffe1-225c-4165-81dd-3b2080e0d0a4 /tidb/cdc/default/default/upstream/7365064565591594020 {"id":7365064565591594020,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1457ffe1-225c-4165-81dd-3b2080e0d0a4 {"id":"1457ffe1-225c-4165-81dd-3b2080e0d0a4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812743} /tidb/cdc/default/__cdc_meta__/capture/9a90ac6a-69c4-4f86-9102-7f5afc873773 {"id":"9a90ac6a-69c4-4f86-9102-7f5afc873773","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812740} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96123a 9a90ac6a-69c4-4f86-9102-7f5afc873773 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96126c 1457ffe1-225c-4165-81dd-3b2080e0d0a4 /tidb/cdc/default/default/upstream/7365064565591594020 {"id":7365064565591594020,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sat May 4 16:52:25 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.88808882.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } [Pipeline] // podTemplate ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] } [Pipeline] // withCredentials VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37c55c0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5, pid:1278, start at 2024-05-04 16:52:24.024666805 +0800 CST m=+6.040729757 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:24.035 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:24.023 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:24.023 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b37c6ec0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-5v8-xq7j5, pid:1355, start at 2024-05-04 16:52:24.124322208 +0800 CST m=+6.061063904 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:24.134 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:24.123 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:24.123 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2651.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } [Pipeline] // timeout table ddl_manager.finish_mark not exists for 29-th check, retry later + set +x check_no_changefeed 127.0.0.1:2379 [Sat May 4 16:52:27 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.26272629.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Sat May 4 16:52:26 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.26322634.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } [Pipeline] // stage [Pipeline] } parse error: Invalid numeric literal at line 1, column 6 run task successfully table drop_tables.c exists check diff successfully [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] } + set +x + tso='449527872641564673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527872641564673 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 16:52:28 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.26942696.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 8-th time, retry later [Pipeline] // withEnv wait process cdc.test exit for 1-th time... [Pipeline] } wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // podTemplate [Sat May 4 16:52:28 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.26082610.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] } wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3800cc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0, pid:1353, start at 2024-05-04 16:52:27.860477522 +0800 CST m=+7.256848507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:27.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:27.827 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:27.827 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3800cc0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0, pid:1353, start at 2024-05-04 16:52:27.860477522 +0800 CST m=+7.256848507 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:27.868 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:27.827 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:27.827 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3803e80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0, pid:1419, start at 2024-05-04 16:52:28.055967568 +0800 CST m=+7.391389682 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:28.066 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:28.026 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:28.026 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:52:29 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> table ddl_manager.finish_mark not exists for 30-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 16:52:29 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + etcd_info_msg='etcd info' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.58635865.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:28 GMT < Content-Length: 1756 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1457ffe1-225c-4165-81dd-3b2080e0d0a4 {"id":"1457ffe1-225c-4165-81dd-3b2080e0d0a4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812743} /tidb/cdc/default/__cdc_meta__/capture/9a90ac6a-69c4-4f86-9102-7f5afc873773 {"id":"9a90ac6a-69c4-4f86-9102-7f5afc873773","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812740} /tidb/cdc/default/__cdc_meta__/capture/c5b2b231-46a1-4cf3-860f-293828ef8860 {"id":"c5b2b231-46a1-4cf3-860f-293828ef8860","address":"127.0.0.1:8303","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812746} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96123a 9a90ac6a-69c4-4f86-9102-7f5afc873773 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96126c 1457ffe1-225c-4165-81dd-3b2080e0d0a4 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96128f c5b2b231-46a1-4cf3-860f-293828ef8860 /tidb/cdc/default/default/upstream/7365064565591594020 {"id":7365064565591594020,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1457ffe1-225c-4165-81dd-3b2080e0d0a4 {"id":"1457ffe1-225c-4165-81dd-3b2080e0d0a4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812743} /tidb/cdc/default/__cdc_meta__/capture/9a90ac6a-69c4-4f86-9102-7f5afc873773 {"id":"9a90ac6a-69c4-4f86-9102-7f5afc873773","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812740} /tidb/cdc/default/__cdc_meta__/capture/c5b2b231-46a1-4cf3-860f-293828ef8860 {"id":"c5b2b231-46a1-4cf3-860f-293828ef8860","address":"127.0.0.1:8303","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812746} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96123a 9a90ac6a-69c4-4f86-9102-7f5afc873773 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96126c 1457ffe1-225c-4165-81dd-3b2080e0d0a4 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96128f c5b2b231-46a1-4cf3-860f-293828ef8860 /tidb/cdc/default/default/upstream/7365064565591594020 {"id":7365064565591594020,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1457ffe1-225c-4165-81dd-3b2080e0d0a4 {"id":"1457ffe1-225c-4165-81dd-3b2080e0d0a4","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812743} /tidb/cdc/default/__cdc_meta__/capture/9a90ac6a-69c4-4f86-9102-7f5afc873773 {"id":"9a90ac6a-69c4-4f86-9102-7f5afc873773","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812740} /tidb/cdc/default/__cdc_meta__/capture/c5b2b231-46a1-4cf3-860f-293828ef8860 {"id":"c5b2b231-46a1-4cf3-860f-293828ef8860","address":"127.0.0.1:8303","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812746} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96123a 9a90ac6a-69c4-4f86-9102-7f5afc873773 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96126c 1457ffe1-225c-4165-81dd-3b2080e0d0a4 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd96128f c5b2b231-46a1-4cf3-860f-293828ef8860 /tidb/cdc/default/default/upstream/7365064565591594020 {"id":7365064565591594020,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8934.out cli changefeed create --start-ts=449527870236393473 '--sink-uri=file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-5395?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 Create changefeed successfully! ID: b7a058c2-8f86-4db6-8bba-7d253ec76809 Info: {"upstream_id":7365064565591594020,"namespace":"default","id":"b7a058c2-8f86-4db6-8bba-7d253ec76809","sink_uri":"file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-5395?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:29.417735394+08:00","start_ts":449527870236393473,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527870236393473,"checkpoint_ts":449527870236393473,"checkpoint_time":"2024-05-04 16:52:17.413"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3814f80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f, pid:5097, start at 2024-05-04 16:52:29.166463015 +0800 CST m=+5.449694429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:29.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:29.168 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:29.168 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:29 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/842b450d-5568-4144-bb27-2107233b57ce {"id":"842b450d-5568-4144-bb27-2107233b57ce","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812747} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc41dcc 842b450d-5568-4144-bb27-2107233b57ce /tidb/cdc/default/default/upstream/7365064607076285542 {"id":7365064607076285542,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/842b450d-5568-4144-bb27-2107233b57ce {"id":"842b450d-5568-4144-bb27-2107233b57ce","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812747} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc41dcc 842b450d-5568-4144-bb27-2107233b57ce /tidb/cdc/default/default/upstream/7365064607076285542 {"id":7365064607076285542,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/842b450d-5568-4144-bb27-2107233b57ce {"id":"842b450d-5568-4144-bb27-2107233b57ce","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812747} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc41dcc 842b450d-5568-4144-bb27-2107233b57ce /tidb/cdc/default/default/upstream/7365064607076285542 {"id":7365064607076285542,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 9-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + workdir=/tmp/tidb_cdc_test/changefeed_finish + sink_uri='file:///tmp/tidb_cdc_test/changefeed_finish/storage_test/ticdc-changefeed-pause-resume-9771?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_finish ++ date + echo '[Sat May 4 16:52:30 CST 2024] <<<<<< START storage consumer in changefeed_finish case >>>>>>' [Sat May 4 16:52:30 CST 2024] <<<<<< START storage consumer in changefeed_finish case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_finish + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_finish + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_finish/storage_test/ticdc-changefeed-pause-resume-9771?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:30 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d45699dd-9ccd-4d03-989c-737442e9f293 {"id":"d45699dd-9ccd-4d03-989c-737442e9f293","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812747} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc307cc d45699dd-9ccd-4d03-989c-737442e9f293 /tidb/cdc/default/default/upstream/7365064611229158930 {"id":7365064611229158930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d45699dd-9ccd-4d03-989c-737442e9f293 {"id":"d45699dd-9ccd-4d03-989c-737442e9f293","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812747} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc307cc d45699dd-9ccd-4d03-989c-737442e9f293 /tidb/cdc/default/default/upstream/7365064611229158930 {"id":7365064611229158930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d45699dd-9ccd-4d03-989c-737442e9f293 {"id":"d45699dd-9ccd-4d03-989c-737442e9f293","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812747} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc307cc d45699dd-9ccd-4d03-989c-737442e9f293 /tidb/cdc/default/default/upstream/7365064611229158930 {"id":7365064611229158930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2716.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 check diff failed 1-th time, retry later Create changefeed successfully! ID: 071ecafa-f7a5-47c8-b89b-b6da3b71166f Info: {"upstream_id":7365064611229158930,"namespace":"default","id":"071ecafa-f7a5-47c8-b89b-b6da3b71166f","sink_uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:30.843644409+08:00","start_ts":449527873725530116,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527873725530116,"checkpoint_ts":449527873725530116,"checkpoint_time":"2024-05-04 16:52:30.723"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + workdir=/tmp/tidb_cdc_test/multi_capture + sink_uri='file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-5395?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/multi_capture ++ date + echo '[Sat May 4 16:52:30 CST 2024] <<<<<< START storage consumer in multi_capture case >>>>>>' [Sat May 4 16:52:30 CST 2024] <<<<<< START storage consumer in multi_capture case >>>>>> + cd /tmp/tidb_cdc_test/multi_capture + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/multi_capture + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/multi_capture/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-5395?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table multi_capture_1.usertable not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 31-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:31 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e79e1d83-ab58-4a28-8392-f5e519fdce9b {"id":"e79e1d83-ab58-4a28-8392-f5e519fdce9b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdbc0cd5 e79e1d83-ab58-4a28-8392-f5e519fdce9b /tidb/cdc/default/default/upstream/7365064604258557797 {"id":7365064604258557797,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e79e1d83-ab58-4a28-8392-f5e519fdce9b {"id":"e79e1d83-ab58-4a28-8392-f5e519fdce9b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdbc0cd5 e79e1d83-ab58-4a28-8392-f5e519fdce9b /tidb/cdc/default/default/upstream/7365064604258557797 {"id":7365064604258557797,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e79e1d83-ab58-4a28-8392-f5e519fdce9b {"id":"e79e1d83-ab58-4a28-8392-f5e519fdce9b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdbc0cd5 e79e1d83-ab58-4a28-8392-f5e519fdce9b /tidb/cdc/default/default/upstream/7365064604258557797 {"id":7365064604258557797,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 1e62ff98-5900-4b68-8d78-20a6f5b594d8 Info: {"upstream_id":7365064604258557797,"namespace":"default","id":"1e62ff98-5900-4b68-8d78-20a6f5b594d8","sink_uri":"file:///tmp/tidb_cdc_test/tiflash/storage_test/ticdc-tiflash-test-894?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:31.359004465+08:00","start_ts":449527872641564673,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527872641564673,"checkpoint_ts":449527872641564673,"checkpoint_time":"2024-05-04 16:52:26.588"} + workdir=/tmp/tidb_cdc_test/tiflash + sink_uri='file:///tmp/tidb_cdc_test/tiflash/storage_test/ticdc-tiflash-test-894?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/tiflash ++ date + echo '[Sat May 4 16:52:31 CST 2024] <<<<<< START storage consumer in tiflash case >>>>>>' [Sat May 4 16:52:31 CST 2024] <<<<<< START storage consumer in tiflash case >>>>>> + cd /tmp/tidb_cdc_test/tiflash + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/tiflash + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/tiflash/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/tiflash/storage_test/ticdc-tiflash-test-894?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:31 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/903024ab-ed4e-4ac7-b82e-670f1bd30170 {"id":"903024ab-ed4e-4ac7-b82e-670f1bd30170","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc5c4cc 903024ab-ed4e-4ac7-b82e-670f1bd30170 /tidb/cdc/default/default/upstream/7365064612290581739 {"id":7365064612290581739,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/903024ab-ed4e-4ac7-b82e-670f1bd30170 {"id":"903024ab-ed4e-4ac7-b82e-670f1bd30170","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc5c4cc 903024ab-ed4e-4ac7-b82e-670f1bd30170 /tidb/cdc/default/default/upstream/7365064612290581739 {"id":7365064612290581739,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/903024ab-ed4e-4ac7-b82e-670f1bd30170 {"id":"903024ab-ed4e-4ac7-b82e-670f1bd30170","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc5c4cc 903024ab-ed4e-4ac7-b82e-670f1bd30170 /tidb/cdc/default/default/upstream/7365064612290581739 {"id":7365064612290581739,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.2667.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: 8cf60ac1-1a70-471a-a102-0e500f40bfff Info: {"upstream_id":7365064612290581739,"namespace":"default","id":"8cf60ac1-1a70-471a-a102-0e500f40bfff","sink_uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:31.952380381+08:00","start_ts":449527874014150663,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527874014150663,"checkpoint_ts":449527874014150663,"checkpoint_time":"2024-05-04 16:52:31.824"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + workdir=/tmp/tidb_cdc_test/owner_resign + sink_uri='file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/owner_resign ++ date + echo '[Sat May 4 16:52:32 CST 2024] <<<<<< START storage consumer in owner_resign case >>>>>>' [Sat May 4 16:52:32 CST 2024] <<<<<< START storage consumer in owner_resign case >>>>>> + cd /tmp/tidb_cdc_test/owner_resign + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/owner_resign + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/owner_resign/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3814f80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f, pid:5097, start at 2024-05-04 16:52:29.166463015 +0800 CST m=+5.449694429 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:29.176 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:29.168 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:29.168 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3816740016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f, pid:5171, start at 2024-05-04 16:52:29.260988322 +0800 CST m=+5.487173345 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:29.270 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:29.263 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:29.263 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/csv_storage_basic/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/csv_storage_basic/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3826b40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj, pid:1382, start at 2024-05-04 16:52:30.277718051 +0800 CST m=+6.144903076 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:30.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:30.253 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:30.253 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3826b40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj, pid:1382, start at 2024-05-04 16:52:30.277718051 +0800 CST m=+6.144903076 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:30.285 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:30.253 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:30.253 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b381e1c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-rlk-pcslj, pid:1462, start at 2024-05-04 16:52:29.729142537 +0800 CST m=+5.511505622 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:29.739 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:29.703 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:29.703 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:32 GMT < Content-Length: 885 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53b728d3-f6c8-4b77-8c2f-8e4a56e3aef2 {"id":"53b728d3-f6c8-4b77-8c2f-8e4a56e3aef2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812750} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f2959 53b728d3-f6c8-4b77-8c2f-8e4a56e3aef2 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53b728d3-f6c8-4b77-8c2f-8e4a56e3aef2 {"id":"53b728d3-f6c8-4b77-8c2f-8e4a56e3aef2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812750} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f2959 53b728d3-f6c8-4b77-8c2f-8e4a56e3aef2 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53b728d3-f6c8-4b77-8c2f-8e4a56e3aef2 {"id":"53b728d3-f6c8-4b77-8c2f-8e4a56e3aef2","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812750} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f2959 53b728d3-f6c8-4b77-8c2f-8e4a56e3aef2 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5914.out cli changefeed create --start-ts=449527863329423361 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-1 table ddl_manager.finish_mark not exists for 32-th check, retry later check diff failed 2-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3846fc000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l, pid:1378, start at 2024-05-04 16:52:32.337061594 +0800 CST m=+5.422371411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:32.345 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:32.319 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:32.319 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365064454781274033,"namespace":"default","id":"changefeed-error-1","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:33.287165284+08:00","start_ts":449527863329423361,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527863329423361,"checkpoint_ts":449527863329423361,"checkpoint_time":"2024-05-04 16:51:51.065"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table multi_capture_1.usertable not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b381e380013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx, pid:1381, start at 2024-05-04 16:52:29.728958184 +0800 CST m=+5.324859417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:29.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:29.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:29.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b381e380013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx, pid:1381, start at 2024-05-04 16:52:29.728958184 +0800 CST m=+5.324859417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:29.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:29.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:29.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b381fa40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx, pid:1470, start at 2024-05-04 16:52:29.823999584 +0800 CST m=+5.366883035 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:29.833 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:29.801 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:29.801 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later + set +x + workdir=/tmp/tidb_cdc_test/processor_resolved_ts_fallback + sink_uri='file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_resolved_ts_fallback ++ date + echo '[Sat May 4 16:52:33 CST 2024] <<<<<< START storage consumer in processor_resolved_ts_fallback case >>>>>>' [Sat May 4 16:52:33 CST 2024] <<<<<< START storage consumer in processor_resolved_ts_fallback case >>>>>> + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later table cdc_tiflash_test.multi_data_type not exists for 2-th check, retry later [Sat May 4 16:52:34 CST 2024] <<<<<< START cdc server in csv_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.64026404.out server --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2721.out cli tso query --pd=http://127.0.0.1:2379 table owner_resign.t1 not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 33-th check, retry later check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3846fc000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l, pid:1378, start at 2024-05-04 16:52:32.337061594 +0800 CST m=+5.422371411 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:32.345 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:32.319 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:32.319 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b384878000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l, pid:1461, start at 2024-05-04 16:52:32.433832458 +0800 CST m=+5.461903747 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:32.441 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:32.414 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:32.414 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x table multi_capture_1.usertable not exists for 3-th check, retry later check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 2147 0 --:--:-- --:--:-- --:--:-- 2162 + info='{"state":"warning","resolved_ts":449527863722377231,"checkpoint_ts":449527863722377231,"last_warning":{"time":"2024-05-04T16:52:34.727454798+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449527863722377231,"checkpoint_ts":449527863722377231,"last_warning":{"time":"2024-05-04T16:52:34.727454798+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449527863722377231,"checkpoint_ts":449527863722377231,"last_warning":{"time":"2024-05-04T16:52:34.727454798+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ jq -r .state ++ echo '{"state":"warning","resolved_ts":449527863722377231,"checkpoint_ts":449527863722377231,"last_warning":{"time":"2024-05-04T16:52:34.727454798+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2736.out cli tso query --pd=http://127.0.0.1:2379 + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449527863722377231,"checkpoint_ts":449527863722377231,"last_warning":{"time":"2024-05-04T16:52:34.727454798+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5963.out cli changefeed remove -c changefeed-error-1 table cdc_tiflash_test.multi_data_type not exists for 3-th check, retry later check diff failed 2-th time, retry later table processor_resolved_ts_fallback.t1 not exists for 2-th check, retry later + set +x + tso='449527874795339778 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527874795339778 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 16:52:36 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.27612763.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449527863722377231 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 4-th time, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.2698.out cli tso query --pd=http://127.0.0.1:2379 table owner_resign.t1 not exists for 3-th check, retry later table multi_capture_1.usertable not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 34-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:37 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab3debc8-ea78-4a22-83b4-90f23a5f7a64 {"id":"ab3debc8-ea78-4a22-83b4-90f23a5f7a64","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812754} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cde210ce ab3debc8-ea78-4a22-83b4-90f23a5f7a64 /tidb/cdc/default/default/upstream/7365064639651473806 {"id":7365064639651473806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab3debc8-ea78-4a22-83b4-90f23a5f7a64 {"id":"ab3debc8-ea78-4a22-83b4-90f23a5f7a64","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812754} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cde210ce ab3debc8-ea78-4a22-83b4-90f23a5f7a64 /tidb/cdc/default/default/upstream/7365064639651473806 {"id":7365064639651473806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ab3debc8-ea78-4a22-83b4-90f23a5f7a64 {"id":"ab3debc8-ea78-4a22-83b4-90f23a5f7a64","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812754} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cde210ce ab3debc8-ea78-4a22-83b4-90f23a5f7a64 /tidb/cdc/default/default/upstream/7365064639651473806 {"id":7365064639651473806,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.cli.6460.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml + set +x + tso='449527875124854786 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527875124854786 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Create changefeed successfully! ID: 515168f7-5e66-4e8b-b968-d7741408c4d3 Info: {"upstream_id":7365064639651473806,"namespace":"default","id":"515168f7-5e66-4e8b-b968-d7741408c4d3","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s","create_time":"2024-05-04T16:52:37.628709188+08:00","start_ts":449527875506798594,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527875506798594,"checkpoint_ts":449527875506798594,"checkpoint_time":"2024-05-04 16:52:37.518"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table cdc_tiflash_test.multi_data_type not exists for 4-th check, retry later + set +x check diff failed 3-th time, retry later table processor_resolved_ts_fallback.t1 not exists for 3-th check, retry later [Sat May 4 16:52:38 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.27272729.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449527875286335491 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527875286335491 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 16:52:38 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.27382740.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 1-th time... [Sat May 4 16:52:38 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.27912793.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 5-th time, retry later + set +x table owner_resign.t1 not exists for 4-th check, retry later wait process cdc.test exit for 2-th time... table multi_capture_1.usertable not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 35-th check, retry later wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:39 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5bb2388-d028-4f52-9411-35909cf03857 {"id":"b5bb2388-d028-4f52-9411-35909cf03857","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812756} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdddb4d4 b5bb2388-d028-4f52-9411-35909cf03857 /tidb/cdc/default/default/upstream/7365064637029676433 {"id":7365064637029676433,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5bb2388-d028-4f52-9411-35909cf03857 {"id":"b5bb2388-d028-4f52-9411-35909cf03857","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812756} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdddb4d4 b5bb2388-d028-4f52-9411-35909cf03857 /tidb/cdc/default/default/upstream/7365064637029676433 {"id":7365064637029676433,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b5bb2388-d028-4f52-9411-35909cf03857 {"id":"b5bb2388-d028-4f52-9411-35909cf03857","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812756} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdddb4d4 b5bb2388-d028-4f52-9411-35909cf03857 /tidb/cdc/default/default/upstream/7365064637029676433 {"id":7365064637029676433,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2818.out cli changefeed create --start-ts=449527874795339778 '--sink-uri=file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-421?protocol=canal-json&enable-tidb-extension=true' table cdc_tiflash_test.multi_data_type not exists for 5-th check, retry later check diff failed 4-th time, retry later table processor_resolved_ts_fallback.t1 not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sat May 4 16:52:40 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.60346036.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Create changefeed successfully! ID: 35d5c829-8d90-419c-b0a4-e7b74bd6d48f Info: {"upstream_id":7365064637029676433,"namespace":"default","id":"35d5c829-8d90-419c-b0a4-e7b74bd6d48f","sink_uri":"file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-421?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:40.40260613+08:00","start_ts":449527874795339778,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527874795339778,"checkpoint_ts":449527874795339778,"checkpoint_time":"2024-05-04 16:52:34.804"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table owner_resign.t1 not exists for 5-th check, retry later table multi_capture_1.usertable not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 36-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m check diff successfully [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc already exists) + workdir=/tmp/tidb_cdc_test/csv_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_basic ++ date + echo '[Sat May 4 16:52:41 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>>' [Sat May 4 16:52:41 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml [Pipeline] // cache [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:41 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a6c30e7-ce2b-405d-84e4-c04c36e59702 {"id":"5a6c30e7-ce2b-405d-84e4-c04c36e59702","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812758} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cde0501b 5a6c30e7-ce2b-405d-84e4-c04c36e59702 /tidb/cdc/default/default/upstream/7365064648005549699 {"id":7365064648005549699,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a6c30e7-ce2b-405d-84e4-c04c36e59702 {"id":"5a6c30e7-ce2b-405d-84e4-c04c36e59702","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812758} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cde0501b 5a6c30e7-ce2b-405d-84e4-c04c36e59702 /tidb/cdc/default/default/upstream/7365064648005549699 {"id":7365064648005549699,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5a6c30e7-ce2b-405d-84e4-c04c36e59702 {"id":"5a6c30e7-ce2b-405d-84e4-c04c36e59702","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812758} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cde0501b 5a6c30e7-ce2b-405d-84e4-c04c36e59702 /tidb/cdc/default/default/upstream/7365064648005549699 {"id":7365064648005549699,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.2785.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-17856?protocol=canal-json&enable-tidb-extension=true' [Pipeline] // dir + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:41 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c5d5dbcf-0fde-4faf-8de7-114fd93e7397 {"id":"c5d5dbcf-0fde-4faf-8de7-114fd93e7397","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812758} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdcc0807 c5d5dbcf-0fde-4faf-8de7-114fd93e7397 /tidb/cdc/default/default/upstream/7365064616654634099 {"id":7365064616654634099,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c5d5dbcf-0fde-4faf-8de7-114fd93e7397 {"id":"c5d5dbcf-0fde-4faf-8de7-114fd93e7397","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812758} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdcc0807 c5d5dbcf-0fde-4faf-8de7-114fd93e7397 /tidb/cdc/default/default/upstream/7365064616654634099 {"id":7365064616654634099,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c5d5dbcf-0fde-4faf-8de7-114fd93e7397 {"id":"c5d5dbcf-0fde-4faf-8de7-114fd93e7397","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812758} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdcc0807 c5d5dbcf-0fde-4faf-8de7-114fd93e7397 /tidb/cdc/default/default/upstream/7365064616654634099 {"id":7365064616654634099,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout Create changefeed successfully! ID: 0a27a1ff-f586-44f9-9fad-58d5cf68f25e Info: {"upstream_id":7365064648005549699,"namespace":"default","id":"0a27a1ff-f586-44f9-9fad-58d5cf68f25e","sink_uri":"file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-17856?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:41.765937277+08:00","start_ts":449527876594958342,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527876594958342,"checkpoint_ts":449527876594958342,"checkpoint_time":"2024-05-04 16:52:41.669"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Pipeline] } + set +x + workdir=/tmp/tidb_cdc_test/resolve_lock + sink_uri='file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-421?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/resolve_lock ++ date + echo '[Sat May 4 16:52:41 CST 2024] <<<<<< START storage consumer in resolve_lock case >>>>>>' [Sat May 4 16:52:41 CST 2024] <<<<<< START storage consumer in resolve_lock case >>>>>> + cd /tmp/tidb_cdc_test/resolve_lock + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/resolve_lock + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/resolve_lock/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-421?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b [Pipeline] // stage table processor_resolved_ts_fallback.t1 not exists for 5-th check, retry later [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:41 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58baeb33-1a38-4499-aafd-7c98a514a098 {"id":"58baeb33-1a38-4499-aafd-7c98a514a098","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812758} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cde2d0f6 58baeb33-1a38-4499-aafd-7c98a514a098 /tidb/cdc/default/default/upstream/7365064643504809123 {"id":7365064643504809123,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58baeb33-1a38-4499-aafd-7c98a514a098 {"id":"58baeb33-1a38-4499-aafd-7c98a514a098","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812758} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cde2d0f6 58baeb33-1a38-4499-aafd-7c98a514a098 /tidb/cdc/default/default/upstream/7365064643504809123 {"id":7365064643504809123,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/58baeb33-1a38-4499-aafd-7c98a514a098 {"id":"58baeb33-1a38-4499-aafd-7c98a514a098","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812758} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cde2d0f6 58baeb33-1a38-4499-aafd-7c98a514a098 /tidb/cdc/default/default/upstream/7365064643504809123 {"id":7365064643504809123,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2851.out cli changefeed create --start-ts=449527875124854786 '--sink-uri=file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-30628?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 8b108a0e-eebc-4fa4-8d95-b722d622760d Info: {"upstream_id":7365064643504809123,"namespace":"default","id":"8b108a0e-eebc-4fa4-8d95-b722d622760d","sink_uri":"file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-30628?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:42.133694283+08:00","start_ts":449527875124854786,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527875124854786,"checkpoint_ts":449527875124854786,"checkpoint_time":"2024-05-04 16:52:36.061"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Pipeline] // container [Pipeline] } table cdc_tiflash_test.multi_data_type exists check diff successfully go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading github.com/coreos/go-semver v0.3.1 go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/google/btree v1.1.2 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // node go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda go: downloading github.com/beorn7/perks v1.0.1 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/prometheus/procfs v0.13.0 [Pipeline] } check diff failed 5-th time, retry later + workdir=/tmp/tidb_cdc_test/capture_session_done_during_task + sink_uri='file:///tmp/tidb_cdc_test/capture_session_done_during_task/storage_test/ticdc-capture-session-done-during-task-18921?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/capture_session_done_during_task ++ date + echo '[Sat May 4 16:52:42 CST 2024] <<<<<< START storage consumer in capture_session_done_during_task case >>>>>>' [Sat May 4 16:52:42 CST 2024] <<<<<< START storage consumer in capture_session_done_during_task case >>>>>> + cd /tmp/tidb_cdc_test/capture_session_done_during_task + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/capture_session_done_during_task + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/capture_session_done_during_task/storage_test/ticdc-capture-session-done-during-task-18921?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' lease 22318f42cdcc0807 revoked [Pipeline] // podTemplate [Pipeline] } go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/cloudfoundry/gosigar v1.3.6 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda [Pipeline] // withEnv table owner_resign.t1 exists check diff failed 1-th time, retry later [Pipeline] } table multi_capture_1.usertable exists table multi_capture_2.usertable exists [Pipeline] // stage [Pipeline] } table ddl_manager.finish_mark not exists for 37-th check, retry later wait process cdc.test exit for 2-th time... table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/batch_add_table + sink_uri='file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-17856?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/batch_add_table ++ date + echo '[Sat May 4 16:52:43 CST 2024] <<<<<< START storage consumer in batch_add_table case >>>>>>' [Sat May 4 16:52:43 CST 2024] <<<<<< START storage consumer in batch_add_table case >>>>>> + cd /tmp/tidb_cdc_test/batch_add_table + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/batch_add_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/batch_add_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-17856?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' go: downloading golang.org/x/text v0.14.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:43 GMT < Content-Length: 885 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03f18d5e-0ec8-4b3d-b6a6-491b07d56f87 {"id":"03f18d5e-0ec8-4b3d-b6a6-491b07d56f87","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812760} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f29d8 03f18d5e-0ec8-4b3d-b6a6-491b07d56f87 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03f18d5e-0ec8-4b3d-b6a6-491b07d56f87 {"id":"03f18d5e-0ec8-4b3d-b6a6-491b07d56f87","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812760} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f29d8 03f18d5e-0ec8-4b3d-b6a6-491b07d56f87 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/03f18d5e-0ec8-4b3d-b6a6-491b07d56f87 {"id":"03f18d5e-0ec8-4b3d-b6a6-491b07d56f87","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812760} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f29d8 03f18d5e-0ec8-4b3d-b6a6-491b07d56f87 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6090.out cli changefeed create --start-ts=449527863329423361 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-2 table batch_add_table.finish_mark not exists for 1-th check, retry later table capture_session_done_during_task.t exists wait process cdc.test exit for 3-th time... Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365064454781274033,"namespace":"default","id":"changefeed-error-2","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:43.648267719+08:00","start_ts":449527863329423361,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527863329423361,"checkpoint_ts":449527863329423361,"checkpoint_time":"2024-05-04 16:51:51.065"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table processor_resolved_ts_fallback.t1 exists [Sat May 4 16:52:43 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.28172819.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + workdir=/tmp/tidb_cdc_test/simple + sink_uri='file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-30628?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/simple ++ date + echo '[Sat May 4 16:52:43 CST 2024] <<<<<< START storage consumer in simple case >>>>>>' [Sat May 4 16:52:43 CST 2024] <<<<<< START storage consumer in simple case >>>>>> + cd /tmp/tidb_cdc_test/simple + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/simple + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/simple/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-30628?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' succeed to verify meta placement rules ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:52:43 CST 2024] <<<<<< run test case tiflash success! >>>>>> check diff failed 6-th time, retry later go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/golang/snappy v0.0.4 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 table ddl_manager.finish_mark not exists for 38-th check, retry later go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/robfig/cron v1.2.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/xitongsys/parquet-go v1.6.0 table multi_capture_4.usertable not exists for 2-th check, retry later go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/apache/thrift v0.16.0 go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/tklauser/go-sysconf v0.3.12 check diff failed 2-th time, retry later table batch_add_table.finish_mark not exists for 2-th check, retry later go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s + info='{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449527863329423361, "checkpoint_time": "2024-05-04 16:51:51.065", "error": { "time": "2024-05-04T16:52:43.733820333+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449527863329423361 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449527863329423361, "checkpoint_time": "2024-05-04 16:51:51.065", "error": { "time": "2024-05-04T16:52:43.733820333+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449527863329423361 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449527863329423361, "checkpoint_time": "2024-05-04 16:51:51.065", "error": { "time": "2024-05-04T16:52:43.733820333+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449527863329423361 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449527863329423361, '"checkpoint_time":' '"2024-05-04' '16:51:51.065",' '"error":' '{' '"time":' '"2024-05-04T16:52:43.733820333+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449527863329423361 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449527863329423361, '"checkpoint_time":' '"2024-05-04' '16:51:51.065",' '"error":' '{' '"time":' '"2024-05-04T16:52:43.733820333+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449527863329423361 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' ++ jq -r .error.message + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449527863329423361 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449527863329423361 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6173.out cli changefeed remove -c changefeed-error-2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 2-th time, retry later go: downloading github.com/go-logr/logr v1.4.1 go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/stdr v1.2.2 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 go: downloading github.com/DataDog/zstd v1.5.5 check diff failed 2-th time, retry later go: downloading k8s.io/apimachinery v0.28.6 Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449527863329423361 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b table multi_capture_4.usertable not exists for 3-th check, retry later check diff failed 7-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/8cf60ac1-1a70-471a-a102-0e500f40bfff {UpstreamID:7365064612290581739 Namespace:default ID:8cf60ac1-1a70-471a-a102-0e500f40bfff SinkURI:file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:52:31.952380381 +0800 CST StartTs:449527874014150663 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036f5950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527874040365062} {CheckpointTs:449527877749702660 MinTableBarrierTs:449527877749702661 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1467abbe-6603-4d60-87bb-fd8850680a79 {"id":"1467abbe-6603-4d60-87bb-fd8850680a79","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812764} /tidb/cdc/default/__cdc_meta__/capture/903024ab-ed4e-4ac7-b82e-670f1bd30170 {"id":"903024ab-ed4e-4ac7-b82e-670f1bd30170","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc5c4cc 903024ab-ed4e-4ac7-b82e-670f1bd30170 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc5c58a 1467abbe-6603-4d60-87bb-fd8850680a79 /tidb/cdc/default/default/changefeed/info/8cf60ac1-1a70-471a-a102-0e500f40bfff {"upstream-id":7365064612290581739,"namespace":"default","changefeed-id":"8cf60ac1-1a70-471a-a102-0e500f40bfff","sink-uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:52:31.952380381+08:00","start-ts":449527874014150663,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527874040365062} /tidb/cdc/default/default/changefeed/status/8cf60ac1-1a70-471a-a102-0e500f40bfff {"checkpoint-ts":449527877749702660,"min-table-barrier-ts":449527877749702661,"admin-job-type":0} /tidb/cdc/default/default/task/position/1467abbe-6603-4d60-87bb-fd8850680a79/8cf60ac1-1a70-471a-a102-0e500f40bfff {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/903024ab-ed4e-4ac7-b82e-670f1bd30170/8cf60ac1-1a70-471a-a102-0e500f40bfff {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064612290581739 {"id":7365064612290581739,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/8cf60ac1-1a70-471a-a102-0e500f40bfff {UpstreamID:7365064612290581739 Namespace:default ID:8cf60ac1-1a70-471a-a102-0e500f40bfff SinkURI:file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:52:31.952380381 +0800 CST StartTs:449527874014150663 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036f5950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527874040365062} {CheckpointTs:449527877749702660 MinTableBarrierTs:449527877749702661 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1467abbe-6603-4d60-87bb-fd8850680a79 {"id":"1467abbe-6603-4d60-87bb-fd8850680a79","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812764} /tidb/cdc/default/__cdc_meta__/capture/903024ab-ed4e-4ac7-b82e-670f1bd30170 {"id":"903024ab-ed4e-4ac7-b82e-670f1bd30170","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc5c4cc 903024ab-ed4e-4ac7-b82e-670f1bd30170 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc5c58a 1467abbe-6603-4d60-87bb-fd8850680a79 /tidb/cdc/default/default/changefeed/info/8cf60ac1-1a70-471a-a102-0e500f40bfff {"upstream-id":7365064612290581739,"namespace":"default","changefeed-id":"8cf60ac1-1a70-471a-a102-0e500f40bfff","sink-uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:52:31.952380381+08:00","start-ts":449527874014150663,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527874040365062} /tidb/cdc/default/default/changefeed/status/8cf60ac1-1a70-471a-a102-0e500f40bfff {"checkpoint-ts":449527877749702660,"min-table-barrier-ts":449527877749702661,"admin-job-type":0} /tidb/cdc/default/default/task/position/1467abbe-6603-4d60-87bb-fd8850680a79/8cf60ac1-1a70-471a-a102-0e500f40bfff {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/903024ab-ed4e-4ac7-b82e-670f1bd30170/8cf60ac1-1a70-471a-a102-0e500f40bfff {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064612290581739 {"id":7365064612290581739,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/8cf60ac1-1a70-471a-a102-0e500f40bfff {UpstreamID:7365064612290581739 Namespace:default ID:8cf60ac1-1a70-471a-a102-0e500f40bfff SinkURI:file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:52:31.952380381 +0800 CST StartTs:449527874014150663 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0036f5950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527874040365062} {CheckpointTs:449527877749702660 MinTableBarrierTs:449527877749702661 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1467abbe-6603-4d60-87bb-fd8850680a79 {"id":"1467abbe-6603-4d60-87bb-fd8850680a79","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812764} /tidb/cdc/default/__cdc_meta__/capture/903024ab-ed4e-4ac7-b82e-670f1bd30170 {"id":"903024ab-ed4e-4ac7-b82e-670f1bd30170","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812748} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc5c4cc 903024ab-ed4e-4ac7-b82e-670f1bd30170 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc5c58a 1467abbe-6603-4d60-87bb-fd8850680a79 /tidb/cdc/default/default/changefeed/info/8cf60ac1-1a70-471a-a102-0e500f40bfff {"upstream-id":7365064612290581739,"namespace":"default","changefeed-id":"8cf60ac1-1a70-471a-a102-0e500f40bfff","sink-uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-13243?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:52:31.952380381+08:00","start-ts":449527874014150663,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527874040365062} /tidb/cdc/default/default/changefeed/status/8cf60ac1-1a70-471a-a102-0e500f40bfff {"checkpoint-ts":449527877749702660,"min-table-barrier-ts":449527877749702661,"admin-job-type":0} /tidb/cdc/default/default/task/position/1467abbe-6603-4d60-87bb-fd8850680a79/8cf60ac1-1a70-471a-a102-0e500f40bfff {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/903024ab-ed4e-4ac7-b82e-670f1bd30170/8cf60ac1-1a70-471a-a102-0e500f40bfff {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064612290581739 {"id":7365064612290581739,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 39-th check, retry later + set +x table batch_add_table.finish_mark not exists for 3-th check, retry later check diff failed 3-th time, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 3-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 40-th check, retry later table multi_capture_4.usertable not exists for 4-th check, retry later check diff failed 8-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:52:49 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.62106212.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 1-th time, retry later table processor_resolved_ts_fallback.t2 not exists for 2-th check, retry later table batch_add_table.finish_mark not exists for 4-th check, retry later go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading github.com/google/s2a-go v0.1.7 go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 check diff failed 4-th time, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 4-th time, retry later table ddl_manager.finish_mark not exists for 41-th check, retry later table multi_capture_4.usertable not exists for 5-th check, retry later check diff successfully check diff failed 2-th time, retry later table processor_resolved_ts_fallback.t2 not exists for 3-th check, retry later table batch_add_table.finish_mark not exists for 5-th check, retry later check diff failed 5-th time, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 5-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:52 GMT < Content-Length: 885 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11296314-1d3c-4d4a-b2d5-070d8b281719 {"id":"11296314-1d3c-4d4a-b2d5-070d8b281719","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812769} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f2a4c 11296314-1d3c-4d4a-b2d5-070d8b281719 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11296314-1d3c-4d4a-b2d5-070d8b281719 {"id":"11296314-1d3c-4d4a-b2d5-070d8b281719","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812769} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f2a4c 11296314-1d3c-4d4a-b2d5-070d8b281719 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/11296314-1d3c-4d4a-b2d5-070d8b281719 {"id":"11296314-1d3c-4d4a-b2d5-070d8b281719","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812769} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f42cd2f2a4c 11296314-1d3c-4d4a-b2d5-070d8b281719 /tidb/cdc/default/default/upstream/7365064454781274033 {"id":7365064454781274033,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6265.out cli changefeed create --start-ts=0 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true' -c changefeed-initialize-error table ddl_manager.finish_mark not exists for 42-th check, retry later Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365064454781274033,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:52:52.962998595+08:00","start_ts":449527879529660417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527879529660417,"checkpoint_ts":449527879529660417,"checkpoint_time":"2024-05-04 16:52:52.864"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table multi_capture_4.usertable exists check diff successfully ***************** properties ***************** "readproportion"="0" "dotransactions"="false" "mysql.user"="root" "insertproportion"="0" "mysql.host"="127.0.0.1" "readallfields"="true" "recordcount"="20" "requestdistribution"="uniform" "scanproportion"="0" "workload"="core" "threadcount"="2" "updateproportion"="0" "operationcount"="0" "mysql.port"="4000" "mysql.db"="multi_capture_1" ********************************************** Run finished, takes 12.775859ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1732.6, Avg(us): 1092, Min(us): 549, Max(us): 1846, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "threadcount"="2" "mysql.db"="multi_capture_2" "workload"="core" "requestdistribution"="uniform" "recordcount"="20" "operationcount"="0" "readproportion"="0" "mysql.user"="root" "dotransactions"="false" "mysql.host"="127.0.0.1" "mysql.port"="4000" "scanproportion"="0" "readallfields"="true" "updateproportion"="0" "insertproportion"="0" ********************************************** Run finished, takes 13.294405ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1659.1, Avg(us): 1147, Min(us): 453, Max(us): 2716, 95th(us): 3000, 99th(us): 3000 ***************** properties ***************** "operationcount"="0" "updateproportion"="0" "threadcount"="2" "workload"="core" "scanproportion"="0" "mysql.port"="4000" "mysql.db"="multi_capture_3" "readallfields"="true" "dotransactions"="false" "requestdistribution"="uniform" "recordcount"="20" "mysql.host"="127.0.0.1" "insertproportion"="0" "readproportion"="0" "mysql.user"="root" ********************************************** Run finished, takes 10.253324ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2174.7, Avg(us): 943, Min(us): 439, Max(us): 1544, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "mysql.host"="127.0.0.1" "operationcount"="0" "updateproportion"="0" "mysql.db"="multi_capture_4" "requestdistribution"="uniform" "recordcount"="20" "insertproportion"="0" "mysql.port"="4000" "threadcount"="2" "readproportion"="0" "dotransactions"="false" "readallfields"="true" "workload"="core" "mysql.user"="root" "scanproportion"="0" ********************************************** Run finished, takes 11.75115ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1911.3, Avg(us): 1126, Min(us): 546, Max(us): 1830, 95th(us): 2000, 99th(us): 2000 check diff failed 1-th time, retry later table processor_resolved_ts_fallback.t2 not exists for 4-th check, retry later check diff failed 1-th time, retry later table batch_add_table.finish_mark not exists for 6-th check, retry later check diff successfully [Sat May 4 16:52:53 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.31203122.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple2' doesn't exist check data failed 6-th time, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check diff successfully check diff failed 1-th time, retry later + info='{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449527879529660417, "checkpoint_time": "2024-05-04 16:52:52.864", "error": { "time": "2024-05-04T16:52:53.152609155+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449527879529660417, "checkpoint_time": "2024-05-04 16:52:52.864", "error": { "time": "2024-05-04T16:52:53.152609155+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449527879529660417, "checkpoint_time": "2024-05-04 16:52:52.864", "error": { "time": "2024-05-04T16:52:53.152609155+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449527879529660417, '"checkpoint_time":' '"2024-05-04' '16:52:52.864",' '"error":' '{' '"time":' '"2024-05-04T16:52:53.152609155+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later check diff failed 2-th time, retry later table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully check diff failed 2-th time, retry later killing cdc1: 2613 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later check diff failed 4-th time, retry later table batch_add_table.finish_mark exists check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 43-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff failed 2-th time, retry later check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 1-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:52:56 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/071ecafa-f7a5-47c8-b89b-b6da3b71166f {UpstreamID:7365064611229158930 Namespace:default ID:071ecafa-f7a5-47c8-b89b-b6da3b71166f SinkURI:file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:52:30.843644409 +0800 CST StartTs:449527873725530116 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00145f950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527873752006664} {CheckpointTs:449527880305344522 MinTableBarrierTs:449527880305344522 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d45699dd-9ccd-4d03-989c-737442e9f293 {"id":"d45699dd-9ccd-4d03-989c-737442e9f293","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812747} /tidb/cdc/default/__cdc_meta__/capture/f548d535-18f0-4109-8e07-50a827303b8f {"id":"f548d535-18f0-4109-8e07-50a827303b8f","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812773} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc307cc d45699dd-9ccd-4d03-989c-737442e9f293 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc30909 f548d535-18f0-4109-8e07-50a827303b8f /tidb/cdc/default/default/changefeed/info/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"upstream-id":7365064611229158930,"namespace":"default","changefeed-id":"071ecafa-f7a5-47c8-b89b-b6da3b71166f","sink-uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:52:30.843644409+08:00","start-ts":449527873725530116,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527873752006664} /tidb/cdc/default/default/changefeed/status/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"checkpoint-ts":449527880305344522,"min-table-barrier-ts":449527880305344522,"admin-job-type":0} /tidb/cdc/default/default/task/position/d45699dd-9ccd-4d03-989c-737442e9f293/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f548d535-18f0-4109-8e07-50a827303b8f/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064611229158930 {"id":7365064611229158930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/071ecafa-f7a5-47c8-b89b-b6da3b71166f {UpstreamID:7365064611229158930 Namespace:default ID:071ecafa-f7a5-47c8-b89b-b6da3b71166f SinkURI:file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:52:30.843644409 +0800 CST StartTs:449527873725530116 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00145f950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527873752006664} {CheckpointTs:449527880305344522 MinTableBarrierTs:449527880305344522 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d45699dd-9ccd-4d03-989c-737442e9f293 {"id":"d45699dd-9ccd-4d03-989c-737442e9f293","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812747} /tidb/cdc/default/__cdc_meta__/capture/f548d535-18f0-4109-8e07-50a827303b8f {"id":"f548d535-18f0-4109-8e07-50a827303b8f","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812773} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc307cc d45699dd-9ccd-4d03-989c-737442e9f293 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc30909 f548d535-18f0-4109-8e07-50a827303b8f /tidb/cdc/default/default/changefeed/info/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"upstream-id":7365064611229158930,"namespace":"default","changefeed-id":"071ecafa-f7a5-47c8-b89b-b6da3b71166f","sink-uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:52:30.843644409+08:00","start-ts":449527873725530116,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527873752006664} /tidb/cdc/default/default/changefeed/status/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"checkpoint-ts":449527880305344522,"mi+ grep -q 'failed to get info:' n-table-barrier-ts":449527880305344522,"admin-job-type":0} /tidb/cdc/default/default/task/position/d45699dd-9ccd-4d03-989c-737442e9f293/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f548d535-18f0-4109-8e07-50a827303b8f/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064611229158930 {"id":7365064611229158930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/071ecafa-f7a5-47c8-b89b-b6da3b71166f {UpstreamID:7365064611229158930 Namespace:default ID:071ecafa-f7a5-47c8-b89b-b6da3b71166f SinkURI:file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-04 16:52:30.843644409 +0800 CST StartTs:449527873725530116 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00145f950 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-79-gc950cce3a Epoch:449527873752006664} {CheckpointTs:449527880305344522 MinTableBarrierTs:449527880305344522 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d45699dd-9ccd-4d03-989c-737442e9f293 {"id":"d45699dd-9ccd-4d03-989c-737442e9f293","address":"127.0.0.1:8301","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812747} /tidb/cdc/default/__cdc_meta__/capture/f548d535-18f0-4109-8e07-50a827303b8f {"id":"f548d535-18f0-4109-8e07-50a827303b8f","address":"127.0.0.1:8302","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812773} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc307cc d45699dd-9ccd-4d03-989c-737442e9f293 /tidb/cdc/default/__cdc_meta__/owner/22318f42cdc30909 f548d535-18f0-4109-8e07-50a827303b8f /tidb/cdc/default/default/changefeed/info/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"upstream-id":7365064611229158930,"namespace":"default","changefeed-id":"071ecafa-f7a5-47c8-b89b-b6da3b71166f","sink-uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-11708?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-04T16:52:30.843644409+08:00","start-ts":449527873725530116,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-79-gc950cce3a","epoch":449527873752006664} /tidb/cdc/default/default/changefeed/status/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"checkpoint-ts":449527880305344522,"min-table-barrier-ts":449527880305344522,"admin-job-type":0} /tidb/cdc/default/default/task/position/d45699dd-9ccd-4d03-989c-737442e9f293/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/f548d535-18f0-4109-8e07-50a827303b8f/071ecafa-f7a5-47c8-b89b-b6da3b71166f {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365064611229158930 {"id":7365064611229158930,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 check diff failed 3-th time, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 2-th time... check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449527879529660417, "checkpoint_time": "2024-05-04 16:52:52.864", "error": { "time": "2024-05-04T16:52:53.152609155+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449527879529660417, "checkpoint_time": "2024-05-04 16:52:52.864", "error": { "time": "2024-05-04T16:52:53.152609155+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449527879529660417, "checkpoint_time": "2024-05-04 16:52:52.864", "error": { "time": "2024-05-04T16:52:53.152609155+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449527879529660417, '"checkpoint_time":' '"2024-05-04' '16:52:52.864",' '"error":' '{' '"time":' '"2024-05-04T16:52:53.152609155+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:52:57 CST 2024] <<<<<< run test case simple success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:52:57 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 44-th check, retry later check diff failed 5-th time, retry later check diff failed 3-th time, retry later check diff failed 4-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 45-th check, retry later check diff failed 1-th time, retry later check diff failed 6-th time, retry later check diff failed 4-th time, retry later check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449527881535324166, "checkpoint_time": "2024-05-04 16:53:00.515", "error": null }' + echo '{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449527881535324166, "checkpoint_time": "2024-05-04 16:53:00.515", "error": null }' { "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449527881535324166, "checkpoint_time": "2024-05-04 16:53:00.515", "error": null } ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449527881535324166, '"checkpoint_time":' '"2024-05-04' '16:53:00.515",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449527881535324166, '"checkpoint_time":' '"2024-05-04' '16:53:00.515",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6433.out cli changefeed pause -c changefeed-initialize-error check diff failed 5-th time, retry later PASS check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b39e4680013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw, pid:6615, start at 2024-05-04 16:52:58.803564165 +0800 CST m=+5.277662257 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:58.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:58.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:58.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b39e4680013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw, pid:6615, start at 2024-05-04 16:52:58.803564165 +0800 CST m=+5.277662257 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:58.814 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:58.778 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:58.778 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b39e5a40008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-w70-rsbnw, pid:6690, start at 2024-05-04 16:52:58.865136062 +0800 CST m=+5.269415602 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:54:58.871 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:52:58.857 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:42:58.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } coverage: 2.0% of statements in github.com/pingcap/tiflow/... cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later table ddl_manager.finish_mark not exists for 46-th check, retry later check diff failed 2-th time, retry later check diff successfully [Sat May 4 16:53:02 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> wait process cdc.test exit for 1-th time... Verifying downstream PD is started... wait process cdc.test exit for 2-th time... + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check diff successfully wait process cdc.test exit for 1-th time... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 3-th time... + info='{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449527881535324166, "checkpoint_time": "2024-05-04 16:53:00.515", "error": { "time": "2024-05-04T16:52:53.152609155+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449527881535324166, "checkpoint_time": "2024-05-04 16:53:00.515", "error": { "time": "2024-05-04T16:52:53.152609155+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449527881535324166, "checkpoint_time": "2024-05-04 16:53:00.515", "error": { "time": "2024-05-04T16:52:53.152609155+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449527881535324166, '"checkpoint_time":' '"2024-05-04' '16:53:00.515",' '"error":' '{' '"time":' '"2024-05-04T16:52:53.152609155+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449527881535324166, '"checkpoint_time":' '"2024-05-04' '16:53:00.515",' '"error":' '{' '"time":' '"2024-05-04T16:52:53.152609155+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6510.out cli changefeed resume -c changefeed-initialize-error wait process cdc.test exit for 2-th time... check diff failed 3-th time, retry later check diff failed 6-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:53:03 CST 2024] <<<<<< run test case multi_capture success! >>>>>> PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sat May 4 16:53:04 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> [Sat May 4 16:53:03 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.79477949.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 47-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s check diff failed 7-th time, retry later + info='{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449527881535324166, "checkpoint_time": "2024-05-04 16:53:00.515", "error": null }' + echo '{ "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449527881535324166, "checkpoint_time": "2024-05-04 16:53:00.515", "error": null }' { "upstream_id": 7365064454781274033, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449527881535324166, "checkpoint_time": "2024-05-04 16:53:00.515", "error": null } ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449527881535324166, '"checkpoint_time":' '"2024-05-04' '16:53:00.515",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365064454781274033, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449527881535324166, '"checkpoint_time":' '"2024-05-04' '16:53:00.515",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6585.out cli changefeed remove -c changefeed-initialize-error table ddl_manager.finish_mark not exists for 48-th check, retry later check diff failed 4-th time, retry later Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449527882846044167 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-9477?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:53:06 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc109bdb-5531-48ff-8301-5d9ce68b4694 {"id":"bc109bdb-5531-48ff-8301-5d9ce68b4694","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812784} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ce4abacc bc109bdb-5531-48ff-8301-5d9ce68b4694 /tidb/cdc/default/default/upstream/7365064761753711453 {"id":7365064761753711453,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc109bdb-5531-48ff-8301-5d9ce68b4694 {"id":"bc109bdb-5531-48ff-8301-5d9ce68b4694","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812784} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ce4abacc bc109bdb-5531-48ff-8301-5d9ce68b4694 /tidb/cdc/default/default/upstream/7365064761753711453 {"id":7365064761753711453,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc109bdb-5531-48ff-8301-5d9ce68b4694 {"id":"bc109bdb-5531-48ff-8301-5d9ce68b4694","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812784} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ce4abacc bc109bdb-5531-48ff-8301-5d9ce68b4694 /tidb/cdc/default/default/upstream/7365064761753711453 {"id":7365064761753711453,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/processor_stop_delay + sink_uri='file:///tmp/tidb_cdc_test/processor_stop_delay/storage_test/ticdc-processor-stop-delay-25748?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_stop_delay ++ date + echo '[Sat May 4 16:53:07 CST 2024] <<<<<< START storage consumer in processor_stop_delay case >>>>>>' [Sat May 4 16:53:07 CST 2024] <<<<<< START storage consumer in processor_stop_delay case >>>>>> + cd /tmp/tidb_cdc_test/processor_stop_delay + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_stop_delay + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_stop_delay/storage_test/ticdc-processor-stop-delay-25748?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check diff failed 8-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table ddl_manager.finish_mark not exists for 49-th check, retry later check diff failed 5-th time, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' table processor_stop_delay.t not exists for 1-th check, retry later 1 run task successfully check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 50-th check, retry later check diff failed 6-th time, retry later check diff failed 9-th time, retry later table processor_stop_delay.t not exists for 2-th check, retry later check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:53:10 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3a9f0c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27, pid:3931, start at 2024-05-04 16:53:10.759874508 +0800 CST m=+5.505558900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:10.767 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:10.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:10.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 51-th check, retry later check diff successfully table processor_stop_delay.t not exists for 3-th check, retry later check diff failed 7-th time, retry later check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3a9f0c0019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27, pid:3931, start at 2024-05-04 16:53:10.759874508 +0800 CST m=+5.505558900 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:10.767 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:10.773 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:10.773 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3aa090000a Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-n0x-47l27, pid:4004, start at 2024-05-04 16:53:10.828245073 +0800 CST m=+5.503412999 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:10.846 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:10.820 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:10.820 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 52-th check, retry later check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 table processor_stop_delay.t not exists for 4-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Sat May 4 16:53:15 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.53095311.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used check diff failed 2-th time, retry later wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=storage + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 53-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table processor_stop_delay.t not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:53:16 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 54-th check, retry later check diff failed 3-th time, retry later table processor_stop_delay.t exists check diff successfully start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:53:18 GMT < Content-Length: 861 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b55b52b-3c04-4a7f-bd7c-059b55881b49 {"id":"0b55b52b-3c04-4a7f-bd7c-059b55881b49","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812795} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ce7a69a1 0b55b52b-3c04-4a7f-bd7c-059b55881b49 /tidb/cdc/default/default/upstream/7365064813674428882 {"id":7365064813674428882,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b55b52b-3c04-4a7f-bd7c-059b55881b49 {"id":"0b55b52b-3c04-4a7f-bd7c-059b55881b49","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812795} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ce7a69a1 0b55b52b-3c04-4a7f-bd7c-059b55881b49 /tidb/cdc/default/default/upstream/7365064813674428882 {"id":7365064813674428882,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b55b52b-3c04-4a7f-bd7c-059b55881b49 {"id":"0b55b52b-3c04-4a7f-bd7c-059b55881b49","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812795} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ce7a69a1 0b55b52b-3c04-4a7f-bd7c-059b55881b49 /tidb/cdc/default/default/upstream/7365064813674428882 {"id":7365064813674428882,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: ae4e700b-0970-443a-927e-7617fd6400ed Info: {"upstream_id":7365064813674428882,"namespace":"default","id":"ae4e700b-0970-443a-927e-7617fd6400ed","sink_uri":"file:///tmp/tidb_cdc_test/region_merge/storage_test/ticdc-region-merge-test-26975?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:53:18.835008541+08:00","start_ts":449527886308704262,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527886308704262,"checkpoint_ts":449527886308704262,"checkpoint_time":"2024-05-04 16:53:18.724"} + workdir=/tmp/tidb_cdc_test/region_merge + sink_uri='file:///tmp/tidb_cdc_test/region_merge/storage_test/ticdc-region-merge-test-26975?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/region_merge ++ date + echo '[Sat May 4 16:53:18 CST 2024] <<<<<< START storage consumer in region_merge case >>>>>>' [Sat May 4 16:53:18 CST 2024] <<<<<< START storage consumer in region_merge case >>>>>> + cd /tmp/tidb_cdc_test/region_merge + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/region_merge + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/region_merge/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/region_merge/storage_test/ticdc-region-merge-test-26975?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... split_and_random_merge scale: 20 start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 55-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 56-th check, retry later check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 57-th check, retry later check diff successfully check diff failed 6-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later *************************** 1. row *************************** count(distinct region_id): 1 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... table ddl_manager.finish_mark not exists for 58-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:53:26 CST 2024] <<<<<< run test case owner_resign success! >>>>>> check diff failed 7-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 59-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3ba7500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:10397, start at 2024-05-04 16:53:27.684607591 +0800 CST m=+5.259448711 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:27.693 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:27.686 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:27.686 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1781/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // dir [Pipeline] } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3bb61c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx, pid:3963, start at 2024-05-04 16:53:28.596980972 +0800 CST m=+5.483115625 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:28.606 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:28.583 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:28.583 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } split_and_random_merge scale: 40 [Pipeline] // withEnv [Pipeline] } check diff failed 5-th time, retry later [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 60-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3ba7500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:10397, start at 2024-05-04 16:53:27.684607591 +0800 CST m=+5.259448711 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:27.693 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:27.686 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:27.686 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3ba7f40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-qv0-mbm3n, pid:10468, start at 2024-05-04 16:53:27.705131389 +0800 CST m=+5.221905434 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:27.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:27.677 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:27.677 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3bb61c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx, pid:3963, start at 2024-05-04 16:53:28.596980972 +0800 CST m=+5.483115625 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:28.606 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:28.583 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:28.583 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3bb7ac0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-kh4-j57nx, pid:4045, start at 2024-05-04 16:53:28.727383036 +0800 CST m=+5.556041871 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:28.737 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:28.733 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:28.733 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 6-th time, retry later check diff successfully ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 61-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11711.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.5422.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 7-th time, retry later check diff failed 1-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3bde580014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0, pid:4199, start at 2024-05-04 16:53:31.196937953 +0800 CST m=+5.539788925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:31.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:31.208 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:31.208 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3bde580014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0, pid:4199, start at 2024-05-04 16:53:31.196937953 +0800 CST m=+5.539788925 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:31.204 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:31.208 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:31.208 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3be0900005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pb1-2jsh0, pid:4265, start at 2024-05-04 16:53:31.303864625 +0800 CST m=+5.570110874 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:31.311 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:31.300 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:31.300 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 62-th check, retry later + set +x + tso='449527890073616385 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527890073616385 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449527890073616385 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sat May 4 16:53:34 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1174811750.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3bf8f00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f, pid:7772, start at 2024-05-04 16:53:32.865179893 +0800 CST m=+6.456231972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:32.874 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:32.860 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:32.860 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3bf8f00005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f, pid:7772, start at 2024-05-04 16:53:32.865179893 +0800 CST m=+6.456231972 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:32.874 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:32.860 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:32.860 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3bf99c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-pqs-v2h1f, pid:7829, start at 2024-05-04 16:53:32.950722865 +0800 CST m=+6.473118746 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:32.960 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:32.952 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:32.952 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449527890256330753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527890256330753 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Sat May 4 16:53:35 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.54635465.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 8-th time, retry later *************************** 1. row *************************** count(distinct region_id): 4 table ddl_manager.finish_mark not exists for 63-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.5569.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sat May 4 16:53:37 CST 2024] <<<<<< START cdc server in csv_storage_multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_multi_tables_ddl.91079109.out server --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449527890931351553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527890931351553 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 16:53:37 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.55935595.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 3-th time, retry later check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:53:37 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d7e754c-6ea8-4c72-b008-597ce97ece90 {"id":"1d7e754c-6ea8-4c72-b008-597ce97ece90","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812814} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cec770d3 1d7e754c-6ea8-4c72-b008-597ce97ece90 /tidb/cdc/default/default/upstream/7365064891403234674 {"id":7365064891403234674,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d7e754c-6ea8-4c72-b008-597ce97ece90 {"id":"1d7e754c-6ea8-4c72-b008-597ce97ece90","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812814} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cec770d3 1d7e754c-6ea8-4c72-b008-597ce97ece90 /tidb/cdc/default/default/upstream/7365064891403234674 {"id":7365064891403234674,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1d7e754c-6ea8-4c72-b008-597ce97ece90 {"id":"1d7e754c-6ea8-4c72-b008-597ce97ece90","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812814} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cec770d3 1d7e754c-6ea8-4c72-b008-597ce97ece90 /tidb/cdc/default/default/upstream/7365064891403234674 {"id":7365064891403234674,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449527890073616385 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11811.out cli changefeed create --start-ts=449527890073616385 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365064891403234674,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-04T16:53:38.098825218+08:00","start_ts":449527890073616385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527890073616385,"checkpoint_ts":449527890073616385,"checkpoint_time":"2024-05-04 16:53:33.086"} PASS table ddl_manager.finish_mark not exists for 64-th check, retry later wait process cdc.test exit for 1-th time... coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:53:38 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd910cee-c92a-4038-9ca2-9bb407959b67 {"id":"dd910cee-c92a-4038-9ca2-9bb407959b67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cec417e5 dd910cee-c92a-4038-9ca2-9bb407959b67 /tidb/cdc/default/default/upstream/7365064889546532618 {"id":7365064889546532618,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd910cee-c92a-4038-9ca2-9bb407959b67 {"id":"dd910cee-c92a-4038-9ca2-9bb407959b67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cec417e5 dd910cee-c92a-4038-9ca2-9bb407959b67 /tidb/cdc/default/default/upstream/7365064889546532618 {"id":7365064889546532618,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/dd910cee-c92a-4038-9ca2-9bb407959b67 {"id":"dd910cee-c92a-4038-9ca2-9bb407959b67","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812815} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cec417e5 dd910cee-c92a-4038-9ca2-9bb407959b67 /tidb/cdc/default/default/upstream/7365064889546532618 {"id":7365064889546532618,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3c359c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs, pid:7590, start at 2024-05-04 16:53:36.754601444 +0800 CST m=+5.897970279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:36.762 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:36.743 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:36.743 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3c359c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs, pid:7590, start at 2024-05-04 16:53:36.754601444 +0800 CST m=+5.897970279 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:36.762 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:36.743 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:36.743 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3c35040011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bpq-l63zs, pid:7653, start at 2024-05-04 16:53:36.718071079 +0800 CST m=+5.783217915 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:36.727 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:36.705 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:36.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 3-th time... + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 3681 0 --:--:-- --:--:-- --:--:-- 3745 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-04 16:53:33.086","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-04 16:53:39.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '16:53:33.086","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '16:53:39.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '16:53:33.086","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '16:53:39.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-04 16:53:33.086' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '16:53:33.086","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '16:53:39.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-04' '16:53:33.086","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-04' '16:53:39.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-04 16:53:39' + echo 'sink_checkpoint_ts is 2024-05-04' 16:53:33.086 sink_checkpoint_ts is 2024-05-04 16:53:33.086 ++ date -d '2024-05-04 16:53:33.086' +%s + checkpoint_timestamp=1714812813 ++ date -d '2024-05-04 16:53:39' +%s + current_timestamp=1714812819 + '[' 6 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' check diff failed 4-th time, retry later + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sat May 4 16:53:39 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 65-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:53:40 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/701a00bc-042c-4fc7-9d91-5020ec695e3a {"id":"701a00bc-042c-4fc7-9d91-5020ec695e3a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812817} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cecef2d0 701a00bc-042c-4fc7-9d91-5020ec695e3a /tidb/cdc/default/default/upstream/7365064907982908359 {"id":7365064907982908359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/701a00bc-042c-4fc7-9d91-5020ec695e3a {"id":"701a00bc-042c-4fc7-9d91-5020ec695e3a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812817} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cecef2d0 701a00bc-042c-4fc7-9d91-5020ec695e3a /tidb/cdc/default/default/upstream/7365064907982908359 {"id":7365064907982908359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/701a00bc-042c-4fc7-9d91-5020ec695e3a {"id":"701a00bc-042c-4fc7-9d91-5020ec695e3a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812817} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cecef2d0 701a00bc-042c-4fc7-9d91-5020ec695e3a /tidb/cdc/default/default/upstream/7365064907982908359 {"id":7365064907982908359,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7365064907982908359,"namespace":"default","id":"test-normal","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-19147?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-04T16:53:40.615816926+08:00","start_ts":449527891102531586,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527891102531586,"checkpoint_ts":449527891102531586,"checkpoint_time":"2024-05-04 16:53:37.011"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7365064907982908359,"namespace":"default","id":"test-error-1","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-3319?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-04T16:53:40.814009167+08:00","start_ts":449527891102531586,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527891102531586,"checkpoint_ts":449527891102531586,"checkpoint_time":"2024-05-04 16:53:37.011"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7365064907982908359,"namespace":"default","id":"test-error-2","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-23984?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-04T16:53:41.012587081+08:00","start_ts":449527891102531586,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527891102531586,"checkpoint_ts":449527891102531586,"checkpoint_time":"2024-05-04 16:53:37.011"} + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-19147?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + log_suffix=1 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Sat May 4 16:53:41 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Sat May 4 16:53:41 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer1.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-19147?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-3319?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + log_suffix=2 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Sat May 4 16:53:41 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Sat May 4 16:53:41 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer2.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-3319?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-23984?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml + log_suffix=3 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Sat May 4 16:53:41 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Sat May 4 16:53:41 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer3.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-23984?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:53:40 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64fd9487-4230-4dd2-b376-9b2fe57fbd0f {"id":"64fd9487-4230-4dd2-b376-9b2fe57fbd0f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812818} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ceca42d3 64fd9487-4230-4dd2-b376-9b2fe57fbd0f /tidb/cdc/default/default/upstream/7365064905769360337 {"id":7365064905769360337,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64fd9487-4230-4dd2-b376-9b2fe57fbd0f {"id":"64fd9487-4230-4dd2-b376-9b2fe57fbd0f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812818} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ceca42d3 64fd9487-4230-4dd2-b376-9b2fe57fbd0f /tidb/cdc/default/default/upstream/7365064905769360337 {"id":7365064905769360337,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/64fd9487-4230-4dd2-b376-9b2fe57fbd0f {"id":"64fd9487-4230-4dd2-b376-9b2fe57fbd0f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812818} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ceca42d3 64fd9487-4230-4dd2-b376-9b2fe57fbd0f /tidb/cdc/default/default/upstream/7365064905769360337 {"id":7365064905769360337,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.5653.out cli changefeed create --start-ts=449527890931351553 '--sink-uri=file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-31188?protocol=canal-json&enable-tidb-extension=true' start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... split_and_random_merge scale: 80 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8897.out cli tso query --pd=http://127.0.0.1:2379 Create changefeed successfully! ID: a13eb536-944a-4a9e-8f92-1ec8c68d6a14 Info: {"upstream_id":7365064905769360337,"namespace":"default","id":"a13eb536-944a-4a9e-8f92-1ec8c68d6a14","sink_uri":"file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-31188?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:53:41.430174526+08:00","start_ts":449527890931351553,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527890931351553,"checkpoint_ts":449527890931351553,"checkpoint_time":"2024-05-04 16:53:36.358"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check diff failed 5-th time, retry later table test.t1 exists + sleep 5 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 66-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + tso='449527892290568193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449527892290568193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sat May 4 16:53:43 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.89348936.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + workdir=/tmp/tidb_cdc_test/ddl_attributes + sink_uri='file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-31188?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_attributes ++ date + echo '[Sat May 4 16:53:42 CST 2024] <<<<<< START storage consumer in ddl_attributes case >>>>>>' [Sat May 4 16:53:42 CST 2024] <<<<<< START storage consumer in ddl_attributes case >>>>>> + cd /tmp/tidb_cdc_test/ddl_attributes + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_attributes + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-31188?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 67-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3cb1100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l, pid:3954, start at 2024-05-04 16:53:44.668218331 +0800 CST m=+27.185524196 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:44.675 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:44.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:44.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3cb1100012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l, pid:3954, start at 2024-05-04 16:53:44.668218331 +0800 CST m=+27.185524196 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:44.675 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:44.644 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:44.644 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3b7b980009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-bzq-2s66l, pid:4033, start at 2024-05-04 16:53:24.842659555 +0800 CST m=+7.299141495 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:24.852 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:24.838 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:24.838 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... check diff failed 6-th time, retry later TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:53:46 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fd960b52-b4a6-48f7-9c84-a06c62877637 {"id":"fd960b52-b4a6-48f7-9c84-a06c62877637","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812823} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cee075d4 fd960b52-b4a6-48f7-9c84-a06c62877637 /tidb/cdc/default/default/upstream/7365064927850555502 {"id":7365064927850555502,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fd960b52-b4a6-48f7-9c84-a06c62877637 {"id":"fd960b52-b4a6-48f7-9c84-a06c62877637","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812823} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cee075d4 fd960b52-b4a6-48f7-9c84-a06c62877637 /tidb/cdc/default/default/upstream/7365064927850555502 {"id":7365064927850555502,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fd960b52-b4a6-48f7-9c84-a06c62877637 {"id":"fd960b52-b4a6-48f7-9c84-a06c62877637","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812823} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42cee075d4 fd960b52-b4a6-48f7-9c84-a06c62877637 /tidb/cdc/default/default/upstream/7365064927850555502 {"id":7365064927850555502,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.8986.out cli changefeed create --start-ts=449527892290568193 '--sink-uri=file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-58?protocol=canal-json&enable-tidb-extension=true' table ddl_manager.finish_mark not exists for 68-th check, retry later Create changefeed successfully! ID: 524c04f4-b733-4311-8852-8afcdc5b9421 Info: {"upstream_id":7365064927850555502,"namespace":"default","id":"524c04f4-b733-4311-8852-8afcdc5b9421","sink_uri":"file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-58?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:53:46.52798519+08:00","start_ts":449527892290568193,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527892290568193,"checkpoint_ts":449527892290568193,"checkpoint_time":"2024-05-04 16:53:41.543"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3079 0 --:--:-- --:--:-- --:--:-- 3115 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-04 16:53:45.487","puller_resolved_ts":"2024-05-04 16:53:39.737","last_synced_ts":"2024-05-04 16:53:39.837","now_ts":"2024-05-04 16:53:46.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-04' '16:53:45.487","puller_resolved_ts":"2024-05-04' '16:53:39.737","last_synced_ts":"2024-05-04' '16:53:39.837","now_ts":"2024-05-04' '16:53:46.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-04' '16:53:45.487","puller_resolved_ts":"2024-05-04' '16:53:39.737","last_synced_ts":"2024-05-04' '16:53:39.837","now_ts":"2024-05-04' '16:53:46.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 7-th time, retry later [Sat May 4 16:53:47 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.1664416646.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 *************************** 1. row *************************** count(distinct region_id): 9 + set +x + workdir=/tmp/tidb_cdc_test/ddl_sequence + sink_uri='file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-58?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_sequence ++ date + echo '[Sat May 4 16:53:48 CST 2024] <<<<<< START storage consumer in ddl_sequence case >>>>>>' [Sat May 4 16:53:48 CST 2024] <<<<<< START storage consumer in ddl_sequence case >>>>>> + cd /tmp/tidb_cdc_test/ddl_sequence + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_sequence + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-58?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table ddl_manager.finish_mark not exists for 69-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 8-th time, retry later table ddl_manager.finish_mark not exists for 70-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sat, 04 May 2024 08:53:50 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ac12235-85e7-45b6-b7e2-160fbf65a1c9 {"id":"2ac12235-85e7-45b6-b7e2-160fbf65a1c9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812828} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ceb22d01 2ac12235-85e7-45b6-b7e2-160fbf65a1c9 /tidb/cdc/default/default/upstream/7365064869013196568 {"id":7365064869013196568,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ac12235-85e7-45b6-b7e2-160fbf65a1c9 {"id":"2ac12235-85e7-45b6-b7e2-160fbf65a1c9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812828} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ceb22d01 2ac12235-85e7-45b6-b7e2-160fbf65a1c9 /tidb/cdc/default/default/upstream/7365064869013196568 {"id":7365064869013196568,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2ac12235-85e7-45b6-b7e2-160fbf65a1c9 {"id":"2ac12235-85e7-45b6-b7e2-160fbf65a1c9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-79-gc950cce3a","git-hash":"c950cce3a9b105fd95bb2c788e1ab69ec32e0668","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714812828} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f42ceb22d01 2ac12235-85e7-45b6-b7e2-160fbf65a1c9 /tidb/cdc/default/default/upstream/7365064869013196568 {"id":7365064869013196568,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.16928.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-25849?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 1525b1fe-8091-4ab8-9b87-c294296e7657 Info: {"upstream_id":7365064869013196568,"namespace":"default","id":"1525b1fe-8091-4ab8-9b87-c294296e7657","sink_uri":"file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-25849?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-04T16:53:51.469929936+08:00","start_ts":449527894859579396,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-79-gc950cce3a","resolved_ts":449527894859579396,"checkpoint_ts":449527894859579396,"checkpoint_time":"2024-05-04 16:53:51.343"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table region_merge.t1 exists check diff failed 1-th time, retry later check diff successfully table ddl_manager.finish_mark not exists for 71-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3d07a40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g, pid:4585, start at 2024-05-04 16:53:50.222263914 +0800 CST m=+5.296427070 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:50.232 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:50.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:50.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3d07a40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g, pid:4585, start at 2024-05-04 16:53:50.222263914 +0800 CST m=+5.296427070 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:50.232 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:50.235 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:50.235 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... Aborted by Jenkins Admin Sending interrupt signal to process Killing processes + set +x + workdir=/tmp/tidb_cdc_test/multi_rocks + sink_uri='file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-25849?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/multi_rocks ++ date + echo '[Sat May 4 16:53:52 CST 2024] <<<<<< START storage consumer in multi_rocks case >>>>>>' [Sat May 4 16:53:52 CST 2024] <<<<<< START storage consumer in multi_rocks case >>>>>> + cd /tmp/tidb_cdc_test/multi_rocks + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/multi_rocks + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/multi_rocks/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-25849?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d0b3d09180015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1781-nbj-5tm8g, pid:4658, start at 2024-05-04 16:53:50.320925481 +0800 CST m=+5.339585030 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240504-16:55:50.329 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240504-16:53:50.328 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240504-16:43:50.328 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } kill finished with exit code 0 Sending interrupt signal to process Killing processes ***************** properties ***************** "readproportion"="0" "table"="a1" "requestdistribution"="uniform" "insertproportion"="0" "threadcount"="2" "readallfields"="true" "updateproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "scanproportion"="0" "mysql.db"="multi_rocks" "mysql.user"="root" "operationcount"="0" "workload"="core" "mysql.port"="4000" "recordcount"="1000" ********************************************** {"level":"warn","ts":1714812833.3165479,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0015868c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 check diff failed 1-th time, retry later table ddl_sequence.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes table ddl_manager.finish_mark not exists for 72-th check, retry later table multi_tables_ddl_test.t55 not exists for 1-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 check diff failed 2-th time, retry later Got signal [terminated] to exit. Run finished, takes 2.331967189s INSERT - Takes(s): 2.3, Count: 916, OPS: 395.1, Avg(us): 3451, Min(us): 862, Max(us): 1555360, 95th(us): 3000, 99th(us): 8000 INSERT_ERROR - Takes(s): 0.0, Count: 1, OPS: 7116.3, Avg(us): 596, Min(us): 596, Max(us): 596, 95th(us): 1000, 99th(us): 1000 script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes table ddl_sequence.finish_mark not exists for 2-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes table multi_tables_ddl_test.t55 not exists for 2-th check, retry later script returned exit code 143 table ddl_manager.finish_mark not exists for 73-th check, retry later {"level":"warn","ts":1714812836.353382,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002300fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 check diff failed 3-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_sequence.finish_mark not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later table ddl_manager.finish_mark not exists for 74-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release {"level":"warn","ts":1714812836.9004498,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022bd180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} {"level":"warn","ts":1714812838.9012175,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022bd180/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} script returned exit code 143 table ddl_sequence.finish_mark not exists for 4-th check, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later table ddl_manager.finish_mark not exists for 75-th check, retry later ++ stop_tidb_cluster script returned exit code 143 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 5-th check, retry later {"level":"warn","ts":1714812835.8119812,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020f16c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:52720->127.0.0.1:2379: read: connection reset by peer"} {"level":"warn","ts":1714812837.8119602,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0020f16c0/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714812842.96613,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022268c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes {"level":"warn","ts":1714812843.2347324,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00216d6c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 kill finished with exit code 0 Sending interrupt signal to process Killing processes table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later kill finished with exit code 0 Sending interrupt signal to process Killing processes script returned exit code 143 kill finished with exit code 0 {"level":"warn","ts":1714812844.581338,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022e4700/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"} script returned exit code 143 script returned exit code 143 Killing processes kill finished with exit code 0 Killing processes kill finished with exit code 0 [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] } Cache not saved (inner-step execution failed) [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache Click here to forcibly terminate running steps [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] // cache [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] // dir [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] // withCredentials [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] // podTemplate [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G03' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G05' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G06' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G08' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G09' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G10' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G11' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G12' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G13' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G14' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G15' [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G16' [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G02' [Pipeline] // stage [Pipeline] } Failed in branch Matrix - TEST_GROUP = 'G04' [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline org.jenkinsci.plugins.workflow.actions.ErrorAction$ErrorId: 8fa02fa5-ec1e-4e32-8df8-720e2175a2f4 Finished: ABORTED