Started by user Jenkins Admin Obtained pipelines/pingcap/tiflow/latest/pull_cdc_integration_storage_test.groovy from git https://github.com/PingCAP-QE/ci.git Loading library tipipeline@main Library tipipeline@main is cached. Copying from home. [Pipeline] Start of Pipeline [Pipeline] readJSON [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714 Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-5dfw1-47svv --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0468e59ea05a7e029553ef891a4388486caad3b6" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-5dfw1" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] stage [Pipeline] { (Declarative: Checkout SCM) [Pipeline] checkout The recommended git tool is: git No credentials specified Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-list --no-walk 03312178c534dce949face80c69812d989e55009 # timeout=10 [Pipeline] } [Pipeline] // stage [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] timeout Timeout set to expire in 1 hr 0 min [Pipeline] { [Pipeline] stage [Pipeline] { (Debug info) [Pipeline] sh + printenv PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=ac22c496-299c-476c-9b55-48c9c92d2994 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Debug info BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct _=/usr/bin/printenv POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test HUDSON_URL=https://do.pingcap.net/jenkins/ JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=3 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-5dfw1 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-ec41a9ea2975eb575c6c655de75271b1a69674254216af41486e1697505d3c9f NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-5dfw1 pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz + echo ------------------------- ------------------------- + go env GO111MODULE='' GOARCH='amd64' GOBIN='' GOCACHE='/home/jenkins/.cache/go-build' GOENV='/home/jenkins/.config/go/env' GOEXE='' GOEXPERIMENT='' GOFLAGS='' GOHOSTARCH='amd64' GOHOSTOS='linux' GOINSECURE='' GOMODCACHE='/go/pkg/mod' GONOPROXY='' GONOSUMDB='' GOOS='linux' GOPATH='/go' GOPRIVATE='' GOPROXY='http://goproxy.apps.svc,https://proxy.golang.org,direct' GOROOT='/usr/local/go' GOSUMDB='sum.golang.org' GOTMPDIR='' GOTOOLCHAIN='auto' GOTOOLDIR='/usr/local/go/pkg/tool/linux_amd64' GOVCS='' GOVERSION='go1.21.6' GCCGO='gccgo' GOAMD64='v1' AR='ar' CC='gcc' CXX='g++' CGO_ENABLED='1' GOMOD='/dev/null' GOWORK='' CGO_CFLAGS='-O2 -g' CGO_CPPFLAGS='' CGO_CXXFLAGS='-O2 -g' CGO_FFLAGS='-O2 -g' CGO_LDFLAGS='-O2 -g' PKG_CONFIG='pkg-config' GOGCCFLAGS='-fPIC -m64 -pthread -Wl,--no-gc-sections -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build3938333588=/tmp/go-build -gno-record-gcc-switches' + echo ------------------------- ------------------------- + echo 'debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714 bash' debug command: kubectl -n jenkins-tiflow exec -ti pingcap-tiflow-pull-cdc-integration-storage-test-1801-5df-xc714 bash [Pipeline] container [Pipeline] { [Pipeline] sh + dig github.com ; <<>> DiG 9.18.16 <<>> github.com ;; global options: +cmd ;; Got answer: ;; ->>HEADER<<- opcode: QUERY, status: NOERROR, id: 3685 ;; flags: qr aa rd ra; QUERY: 1, ANSWER: 1, AUTHORITY: 0, ADDITIONAL: 1 ;; OPT PSEUDOSECTION: ; EDNS: version: 0, flags:; udp: 1232 ; COOKIE: 1f25292a4705c987 (echoed) ;; QUESTION SECTION: ;github.com. IN A ;; ANSWER SECTION: github.com. 26 IN A 20.205.243.166 ;; Query time: 0 msec ;; SERVER: 169.254.25.10#53(169.254.25.10) (UDP) ;; WHEN: Sun May 05 04:47:58 UTC 2024 ;; MSG SIZE rcvd: 77 [Pipeline] script [Pipeline] { [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Check diff files) [Pipeline] container [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $token [Pipeline] { [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=1&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] httpRequest Warning: A secret was passed to "httpRequest" using Groovy String interpolation, which is insecure. Affected argument(s) used the following variable(s): [token] See https://jenkins.io/redirect/groovy-string-interpolation for details. HttpMethod: GET URL: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Content-Type: application/json Authorization: ***** Sending request to url: https://api.github.com/repos/pingcap/tiflow/pulls/10919/files?page=2&per_page=100 Response Code: HTTP/1.1 200 OK Success: Status code 200 is in the accepted range: 100:399 [Pipeline] } [Pipeline] // withCredentials [Pipeline] echo pr_diff_files: [cdc/model/kv.go, cdc/model/sink.go, cdc/model/sink_test.go, cdc/processor/processor.go, cdc/processor/sinkmanager/manager.go, cdc/processor/sourcemanager/manager.go, cdc/redo/reader/reader.go, cdc/sink/dmlsink/factory/factory.go, cdc/sink/dmlsink/txn/mysql/mysql.go, cdc/sink/dmlsink/txn/mysql/mysql_test.go, cmd/kafka-consumer/main.go, cmd/pulsar-consumer/main.go, cmd/storage-consumer/main.go, errors.toml, pkg/applier/redo.go, pkg/applier/redo_test.go, pkg/errors/cdc_errors.go, pkg/errors/helper.go, pkg/sink/codec/open/open_protocol_decoder.go, tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml, tests/integration_tests/changefeed_dup_error_restart/conf/workload, tests/integration_tests/changefeed_dup_error_restart/run.sh, tests/integration_tests/force_replicate_table/run.sh, tests/integration_tests/run_group.sh] [Pipeline] echo diff file not matched: cdc/model/kv.go [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Checkout) [Pipeline] timeout Timeout set to expire in 10 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (git/pingcap/tiflow/rev-be15534-0de8dc3) 203830272 bytes in 2.23 secs (91281429 bytes/sec) [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] script [Pipeline] { [Pipeline] sh git version 2.36.6 Reinitialized existing Git repository in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/.git/ .git HEAD is now at 0de8dc3e4 fix test again POST git-upload-pack (656 bytes) From https://github.com/pingcap/tiflow = [up to date] master -> origin/master = [up to date] refs/pull/10919/head -> origin/pr/10919/head Previous HEAD position was 0de8dc3e4 fix test again HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) ๐Ÿšง Checkouting to base SHA:be1553484fe4c03594eabb8d7435c694e5fd7224... HEAD is now at be1553484 codec(ticdc): avro simplify the unit test (#11010) โœ… Checked. ๐ŸŽ‰ ๐Ÿงพ HEAD info: be1553484fe4c03594eabb8d7435c694e5fd7224 be1553484 codec(ticdc): avro simplify the unit test (#11010) 2a7a65c6f Support Sequences (#10203) 36e9e1bf6 cli(ticdc): allow client authentication to be enabled without tls (#11005) ๐Ÿšง Pre-merge heads of pull requests to base SHA: be1553484fe4c03594eabb8d7435c694e5fd7224 ... Updating be1553484..0de8dc3e4 Fast-forward cdc/model/kv.go | 5 + cdc/model/sink.go | 35 ++- cdc/model/sink_test.go | 9 +- cdc/processor/processor.go | 21 +- cdc/processor/sinkmanager/manager.go | 5 + cdc/processor/sourcemanager/manager.go | 66 +++- cdc/redo/reader/reader.go | 21 +- cdc/sink/dmlsink/factory/factory.go | 8 +- cdc/sink/dmlsink/txn/mysql/mysql.go | 89 +++--- cdc/sink/dmlsink/txn/mysql/mysql_test.go | 2 +- cmd/kafka-consumer/main.go | 4 +- cmd/pulsar-consumer/main.go | 17 +- cmd/storage-consumer/main.go | 4 +- errors.toml | 5 + pkg/applier/redo.go | 303 +++++++++++++++++- pkg/applier/redo_test.go | 347 ++++++++++++++++++++- pkg/errors/cdc_errors.go | 4 + pkg/errors/helper.go | 19 ++ pkg/sink/codec/open/open_protocol_decoder.go | 2 + .../conf/diff_config.toml | 29 ++ .../changefeed_dup_error_restart/conf/workload | 13 + .../changefeed_dup_error_restart/run.sh | 54 ++++ .../integration_tests/force_replicate_table/run.sh | 4 +- tests/integration_tests/run_group.sh | 2 +- 24 files changed, 970 insertions(+), 98 deletions(-) create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/diff_config.toml create mode 100644 tests/integration_tests/changefeed_dup_error_restart/conf/workload create mode 100755 tests/integration_tests/changefeed_dup_error_restart/run.sh ๐Ÿงพ Pre-merged result: 0de8dc3e43ec741eba58047155ce7f3dba8eb4f7 0de8dc3e4 fix test again 6a342866d fix bit test 0dd104704 fix โœ… Pre merged ๐ŸŽ‰ โœ… ~~~~~All done.~~~~~~ [Pipeline] } [Pipeline] // script [Pipeline] } [Pipeline] // retry [Pipeline] } Cache not saved (git/pingcap/tiflow/rev-be15534-0de8dc3 already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (prepare) [Pipeline] timeout Timeout set to expire in 20 min [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/third_party_download [Pipeline] { [Pipeline] retry [Pipeline] { [Pipeline] sh + cd ../tiflow + ./scripts/download-integration-test-binaries.sh master Download binaries... % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 790 0 --:--:-- --:--:-- --:--:-- 803 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 564 0 --:--:-- --:--:-- --:--:-- 561 100 41 100 41 0 0 563 0 --:--:-- --:--:-- --:--:-- 561 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 1422 0 --:--:-- --:--:-- --:--:-- 1464 % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 41 100 41 0 0 437 0 --:--:-- --:--:-- --:--:-- 440 >>> download tidb-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz 2024-05-05 12:48:31 URL:http://fileserver.pingcap.net/download/builds/pingcap/tidb/600b2ed4bf0aa38224a1c4c4c68831820735515c/centos7/tidb-server.tar.gz [536570515/536570515] -> "tmp/tidb-server.tar.gz" [1] >>> download pd-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz 2024-05-05 12:48:50 URL:http://fileserver.pingcap.net/download/builds/pingcap/pd/1679dbca25b3483d1375c7e747da27e99ad77360/centos7/pd-server.tar.gz [187372022/187372022] -> "tmp/pd-server.tar.gz" [1] >>> download tikv-server.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz 2024-05-05 12:49:34 URL:http://fileserver.pingcap.net/download/builds/pingcap/tikv/72a0fd5b00235a7c56014b77ddd933e2a0d33c88/centos7/tikv-server.tar.gz [919098782/919098782] -> "tmp/tikv-server.tar.gz" [1] >>> download tiflash.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz 2024-05-05 12:50:02 URL:http://fileserver.pingcap.net/download/builds/pingcap/tiflash/master/8e170090fad91c94bef8d908e21c195c1d145b02/centos7/tiflash.tar.gz [456057803/456057803] -> "tmp/tiflash.tar.gz" [1] >>> download minio.tar.gz from http://fileserver.pingcap.net/download/minio.tar.gz 2024-05-05 12:50:07 URL:http://fileserver.pingcap.net/download/minio.tar.gz [17718777/17718777] -> "tmp/minio.tar.gz" [1] >>> download go-ycsb from http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb 2024-05-05 12:50:10 URL:http://fileserver.pingcap.net/download/builds/pingcap/go-ycsb/test-br/go-ycsb [45975512/45975512] -> "third_bin/go-ycsb" [1] >>> download jq from http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 2024-05-05 12:50:10 URL:http://fileserver.pingcap.net/download/builds/pingcap/test/jq-1.6/jq-linux64 [3953824/3953824] -> "third_bin/jq" [1] >>> download etcd.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz 2024-05-05 12:50:11 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/etcd-v3.4.7-linux-amd64.tar.gz [17310840/17310840] -> "tmp/etcd.tar.gz" [1] >>> download sync_diff_inspector.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz 2024-05-05 12:50:16 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/sync_diff_inspector_hash-d671b084_linux-amd64.tar.gz [79877126/79877126] -> "tmp/sync_diff_inspector.tar.gz" [1] >>> download schema-registry.tar.gz from http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz 2024-05-05 12:50:33 URL:http://fileserver.pingcap.net/download/builds/pingcap/cdc/schema-registry.tar.gz [278386006/278386006] -> "tmp/schema-registry.tar.gz" [1] Download SUCCESS + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 5 12:50 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 12:50 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 5 12:50 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 5 12:50 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + make check_third_party_binary /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/tidb-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/tikv-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/pd-server /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/tiflash /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/pd-ctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/sync_diff_inspector /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/go-ycsb /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/etcdctl /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/jq /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/minio /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/bin/schema-registry-start + cd - /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/third_party_download + mkdir -p bin + mv ../tiflow/bin/bin ../tiflow/bin/etc ../tiflow/bin/etcdctl ../tiflow/bin/go-ycsb ../tiflow/bin/jq ../tiflow/bin/lib ../tiflow/bin/libc++.so.1 ../tiflow/bin/libc++.so.1.0 ../tiflow/bin/libc++abi.so.1 ../tiflow/bin/libc++abi.so.1.0 ../tiflow/bin/libgmssl.so ../tiflow/bin/libgmssl.so.3 ../tiflow/bin/libgmssl.so.3.0 ../tiflow/bin/libtiflash_proxy.so ../tiflow/bin/minio ../tiflow/bin/pd-api-bench ../tiflow/bin/pd-ctl ../tiflow/bin/pd-heartbeat-bench ../tiflow/bin/pd-recover ../tiflow/bin/pd-server ../tiflow/bin/pd-tso-bench ../tiflow/bin/pd-ut ../tiflow/bin/regions-dump ../tiflow/bin/share ../tiflow/bin/stores-dump ../tiflow/bin/sync_diff_inspector ../tiflow/bin/tidb-server ../tiflow/bin/tiflash ../tiflow/bin/tikv-server ../tiflow/bin/xprog ./bin/ + ls -alh ./bin total 1.9G drwxr-sr-x. 6 jenkins jenkins 4.0K May 5 12:50 . drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 12:50 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 19 2023 bin drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 etc -rwxr-xr-x. 1 jenkins jenkins 17M Apr 2 2020 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 5 12:50 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 5 12:50 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 10 2023 lib lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K Nov 7 01:00 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 Apr 30 11:15 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K Nov 7 01:00 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 Apr 30 11:15 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 Apr 30 11:15 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M Apr 30 10:34 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M Apr 30 11:16 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M Jul 29 2020 minio -rwxr-xr-x. 1 jenkins jenkins 37M Apr 30 16:11 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M Apr 30 16:10 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M Apr 30 16:10 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M Apr 30 16:10 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M Apr 30 16:10 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M Apr 30 16:11 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:10 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 10 2023 share -rwxr-xr-x. 1 jenkins jenkins 32M Apr 30 16:11 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M Sep 22 2023 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 1 10:57 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M Apr 30 11:15 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M Apr 30 11:29 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M Apr 30 16:11 xprog + ./bin/tidb-server -V Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore + ./bin/pd-server -V Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 + ./bin/tikv-server -V TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + ./bin/tiflash --version TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored + ./bin/sync_diff_inspector --version App Name: sync_diff_inspector v2.0 Release Version: v7.4.0 Git Commit Hash: d671b0840063bc2532941f02e02e12627402844c Git Branch: heads/refs/tags/v7.4.0 UTC Build Time: 2023-09-22 03:51:56 Go Version: go1.21.1 [Pipeline] } [Pipeline] // retry [Pipeline] } [Pipeline] // dir [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache Cache restored successfully (binary/pingcap/tiflow/cdc-integration-test/rev-be15534-0de8dc3) 1191700480 bytes in 4.38 secs (272037512 bytes/sec) [Pipeline] { [Pipeline] sh + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 12:50 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 12:50 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 4 22:57 cdc -rwxr-xr-x. 1 jenkins jenkins 359M May 4 22:57 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 22:53 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 4 22:53 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 22:52 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 4 22:53 oauth2-server + '[' -f ./bin/cdc ']' + '[' -f ./bin/cdc_kafka_consumer ']' + '[' -f ./bin/cdc_storage_consumer ']' + '[' -f ./bin/cdc.test ']' + ls -alh ./bin total 1.2G drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 12:50 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 12:50 .. -rwxr-xr-x. 1 jenkins jenkins 220M May 4 22:57 cdc -rwxr-xr-x. 1 jenkins jenkins 359M May 4 22:57 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 22:53 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 4 22:53 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 22:52 cdc_storage_consumer -rwxr-xr-x. 1 jenkins jenkins 12M May 4 22:53 oauth2-server + ./bin/cdc version Release Version: v8.2.0-alpha-53-g0de8dc3e4 Git Commit Hash: 0de8dc3e43ec741eba58047155ce7f3dba8eb4f7 Git Branch: HEAD UTC Build Time: 2024-05-04 14:52:44 Go Version: go version go1.21.0 linux/amd64 Failpoint Build: true [Pipeline] } Cache not saved (binary/pingcap/tiflow/cdc-integration-test/rev-be15534-0de8dc3 already exists) [Pipeline] // cache [Pipeline] cache Cache not restored (no such key found) [Pipeline] { [Pipeline] sh + cp -r ../third_party_download/bin/bin ../third_party_download/bin/etc ../third_party_download/bin/etcdctl ../third_party_download/bin/go-ycsb ../third_party_download/bin/jq ../third_party_download/bin/lib ../third_party_download/bin/libc++.so.1 ../third_party_download/bin/libc++.so.1.0 ../third_party_download/bin/libc++abi.so.1 ../third_party_download/bin/libc++abi.so.1.0 ../third_party_download/bin/libgmssl.so ../third_party_download/bin/libgmssl.so.3 ../third_party_download/bin/libgmssl.so.3.0 ../third_party_download/bin/libtiflash_proxy.so ../third_party_download/bin/minio ../third_party_download/bin/pd-api-bench ../third_party_download/bin/pd-ctl ../third_party_download/bin/pd-heartbeat-bench ../third_party_download/bin/pd-recover ../third_party_download/bin/pd-server ../third_party_download/bin/pd-tso-bench ../third_party_download/bin/pd-ut ../third_party_download/bin/regions-dump ../third_party_download/bin/share ../third_party_download/bin/stores-dump ../third_party_download/bin/sync_diff_inspector ../third_party_download/bin/tidb-server ../third_party_download/bin/tiflash ../third_party_download/bin/tikv-server ../third_party_download/bin/xprog ./bin/ + ls -alh ./bin total 3.0G drwxr-sr-x. 6 jenkins jenkins 4.0K May 5 12:50 . drwxr-sr-x. 19 jenkins jenkins 4.0K May 5 12:50 .. drwxr-sr-x. 2 jenkins jenkins 4.0K May 5 12:50 bin -rwxr-xr-x. 1 jenkins jenkins 220M May 4 22:57 cdc -rwxr-xr-x. 1 jenkins jenkins 359M May 4 22:57 cdc.test -rwxr-xr-x. 1 jenkins jenkins 183M May 4 22:53 cdc_kafka_consumer -rwxr-xr-x. 1 jenkins jenkins 183M May 4 22:53 cdc_pulsar_consumer -rwxr-xr-x. 1 jenkins jenkins 182M May 4 22:52 cdc_storage_consumer drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 12:50 etc -rwxr-xr-x. 1 jenkins jenkins 17M May 5 12:50 etcdctl -rwxr-xr-x. 1 jenkins jenkins 44M May 5 12:50 go-ycsb -rwxr-xr-x. 1 jenkins jenkins 3.8M May 5 12:50 jq drwxr-sr-x. 3 jenkins jenkins 4.0K May 5 12:50 lib lrwxrwxrwx. 1 jenkins jenkins 13 May 5 12:50 libc++.so.1 -> libc++.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 1016K May 5 12:50 libc++.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 16 May 5 12:50 libc++abi.so.1 -> libc++abi.so.1.0 -rwxr-xr-x. 1 jenkins jenkins 358K May 5 12:50 libc++abi.so.1.0 lrwxrwxrwx. 1 jenkins jenkins 13 May 5 12:50 libgmssl.so -> libgmssl.so.3 lrwxrwxrwx. 1 jenkins jenkins 15 May 5 12:50 libgmssl.so.3 -> libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 2.6M May 5 12:50 libgmssl.so.3.0 -rwxr-xr-x. 1 jenkins jenkins 272M May 5 12:50 libtiflash_proxy.so -rwxr-xr-x. 1 jenkins jenkins 50M May 5 12:50 minio -rwxr-xr-x. 1 jenkins jenkins 12M May 4 22:53 oauth2-server -rwxr-xr-x. 1 jenkins jenkins 37M May 5 12:50 pd-api-bench -rwxr-xr-x. 1 jenkins jenkins 44M May 5 12:50 pd-ctl -rwxr-xr-x. 1 jenkins jenkins 36M May 5 12:50 pd-heartbeat-bench -rwxr-xr-x. 1 jenkins jenkins 32M May 5 12:50 pd-recover -rwxr-xr-x. 1 jenkins jenkins 106M May 5 12:50 pd-server -rwxr-xr-x. 1 jenkins jenkins 26M May 5 12:50 pd-tso-bench -rwxr-xr-x. 1 jenkins jenkins 3.0M May 5 12:50 pd-ut -rwxr-xr-x. 1 jenkins jenkins 32M May 5 12:50 regions-dump drwxr-sr-x. 4 jenkins jenkins 4.0K May 5 12:50 share -rwxr-xr-x. 1 jenkins jenkins 32M May 5 12:50 stores-dump -rwxr-xr-x. 1 jenkins jenkins 192M May 5 12:50 sync_diff_inspector -rwxr-xr-x. 1 jenkins jenkins 208M May 5 12:50 tidb-server -rwxr-xr-x. 1 jenkins jenkins 380M May 5 12:50 tiflash -rwxr-xr-x. 1 jenkins jenkins 418M May 5 12:50 tikv-server -rwxr-xr-x. 1 jenkins jenkins 2.0M May 5 12:50 xprog [Pipeline] } Cache saved successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 83.48 secs (44605330 bytes/sec) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] stage [Pipeline] { (Tests) [Pipeline] parallel [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G00') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G01') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G02') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G03') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G04') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G05') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G06') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G07') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G08') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G09') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G10') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G11') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G12') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G13') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G14') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G15') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G16') [Pipeline] { (Branch: Matrix - TEST_GROUP = 'G17') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G00') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G01') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G02') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G03') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G04') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G05') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G06') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G07') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G08') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G09') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G10') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G11') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G12') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G13') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G14') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G15') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G16') [Pipeline] stage [Pipeline] { (Matrix - TEST_GROUP = 'G17') [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted [Pipeline] readTrusted Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0 [Pipeline] node Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3 Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-spl28-z17xq --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d1034b4b4dc0e2ca4297013c67b6860ca934c4ac" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-spl28" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@4d1110fb; decorates RemoteLauncher[hudson.remoting.Channel@1d7fecaa:JNLP4-connect connection from 10.233.71.228/10.233.71.228:51340] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc [Pipeline] { [Pipeline] node Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-n3bms-jzgrl --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "acff52fbe3ad253e42cd20faf684adc3444ac1c7" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-n3bms" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] checkout The recommended git tool is: git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate [Pipeline] { [Pipeline] node Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@46ed9487; decorates RemoteLauncher[hudson.remoting.Channel@2e8afc70:JNLP4-connect connection from 10.233.105.133/10.233.105.133:45646] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git [Pipeline] podTemplate Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-8bm7h-mf40h --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "40cc67f9a705d7fc74e47e2d23ae8c7313c9b26f" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-8bm7h" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" [Pipeline] { Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] { [Pipeline] node [Pipeline] checkout The recommended git tool is: git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-69hgf-wjvqc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "31a8488c0e8912c60b6a546abcee0c2ab0581907" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-69hgf" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] checkout [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-r7gz8-4j2kc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "782e7483071b9bf6c4fcd7d6d0d673c8a9d7be7f" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-r7gz8" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" The recommended git tool is: git [Pipeline] { Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test [Pipeline] cache > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@303a08a6; decorates RemoteLauncher[hudson.remoting.Channel@3b2bd729:JNLP4-connect connection from 10.233.86.1/10.233.86.1:52850] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@594b2259; decorates RemoteLauncher[hudson.remoting.Channel@43d0cf:JNLP4-connect connection from 10.233.72.49/10.233.72.49:57022] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-20f8d-d0zm2 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ae28d060c3d0cf22f44bdacd71db78e147a32ff4" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-20f8d" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-xj7hn-w6g7z --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "fadbfdddc3e61fb3e08bda65f8e9707451c205a3" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-xj7hn" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-7f60r-z2067 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "7876d862bfbbf41df54cf6aa77aabfbcb8054984" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-7f60r" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-6sdlp-015td --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "bf7e0859e48eca954de545ed18e4a5b4e3094188" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-6sdlp" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-hpjnh-31bfk --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "3f7b8b87d43abb04083961bdc311d35e414d717d" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-hpjnh" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Obtained pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml from git https://github.com/PingCAP-QE/ci.git Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 19.11 secs (194844678 bytes/sec) [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] podTemplate [Pipeline] { [Pipeline] sh [Pipeline] { [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node [Pipeline] node + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G00 Run cases: bdr_mode capture_suicide_while_balance_table syncpoint hang_sink_suicide server_config_compatibility changefeed_dup_error_restart kafka_big_messages kafka_compression kafka_messages kafka_sink_error_resume mq_sink_lost_callback mq_sink_dispatcher kafka_column_selector kafka_column_selector_avro debezium lossy_ddl storage_csv_update PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=e66b3dc0-a039-482b-a040-7378a79b4824 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G00 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-spl28 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-spl28 pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/bdr_mode/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:52:41 CST 2024] <<<<<< run test case bdr_mode success! >>>>>> [Pipeline] node [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] checkout [Pipeline] container [Pipeline] { The recommended git tool is: git [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@40a00501; decorates RemoteLauncher[hudson.remoting.Channel@4b3487e6:JNLP4-connect connection from 10.233.108.89/10.233.108.89:39058] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] cache Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44 Created Pod: kubernetes jenkins-tiflow/pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6ad413c2; decorates RemoteLauncher[hudson.remoting.Channel@718adc44:JNLP4-connect connection from 10.233.93.33/10.233.93.33:42406] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@75f21da2; decorates RemoteLauncher[hudson.remoting.Channel@7de5d85c:JNLP4-connect connection from 10.233.86.205/10.233.86.205:50910] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@747da8cf; decorates RemoteLauncher[hudson.remoting.Channel@3d77a0d6:JNLP4-connect connection from 10.233.93.32/10.233.93.32:51886] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7f3b4dcf; decorates RemoteLauncher[hudson.remoting.Channel@3f2e1634:JNLP4-connect connection from 10.233.71.106/10.233.71.106:37132] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3e9f137; decorates RemoteLauncher[hudson.remoting.Channel@46e2ab6e:JNLP4-connect connection from 10.233.66.135/10.233.66.135:46820] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/capture_suicide_while_balance_table/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:52:44 CST 2024] <<<<<< run test case capture_suicide_while_balance_table success! >>>>>> Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-k2zh5-rhh90 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "91f4435e0c9e6a39210f656359718305d8de7600" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-k2zh5" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: storage... <<================= kafka downstream isn't support syncpoint record [Sun May 5 12:52:48 CST 2024] <<<<<< run test case syncpoint success! >>>>>> Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-7fvb6-b992j --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ed54cdf14a3bfe146ecefd98429202eacef5df89" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-7fvb6" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-g9d6f-87bj3 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "ffc4f9a4b5cbd03dea5307c6ca21b3ee122bca56" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-g9d6f" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-kkzwj-xdwrc --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "d598bd17b0577bca3024f910f64ddc91b7ce32cf" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-kkzwj" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-59rmb-v2974 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "2131652b33bdcb43252f83b0967c545bf47c865b" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-59rmb" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-gn6zg-1qnp4 --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "0683453fa1adfeb62d9a0d7d84346bc4171a593c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-gn6zg" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/hang_sink_suicide/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:52:51 CST 2024] <<<<<< run test case hang_sink_suicide success! >>>>>> Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-m7bnk-mwwss --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "fb99c45bf901393c3c31b4d0131e2ee0eca9e23c" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-m7bnk" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test Agent pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51 is provisioned from template pingcap_tiflow_pull_cdc_integration_storage_test_1801-sx9p5-mwvcb --- apiVersion: "v1" kind: "Pod" metadata: annotations: buildUrl: "http://jenkins.apps.svc.cluster.local:8080/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" runUrl: "job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/" labels: jenkins/jenkins-jenkins-agent: "true" jenkins/label-digest: "1f9be2125b999824c39a443a4ca0d8c2d54e0c79" jenkins/label: "pingcap_tiflow_pull_cdc_integration_storage_test_1801-sx9p5" name: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51" namespace: "jenkins-tiflow" spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - matchExpressions: - key: "kubernetes.io/arch" operator: "In" values: - "amd64" containers: - image: "hub.pingcap.net/jenkins/centos7_golang-1.21:latest" name: "golang" resources: limits: memory: "16Gi" cpu: "6" requests: memory: "12Gi" cpu: "4" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/network-multitool" name: "net-tool" resources: limits: memory: "128Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - image: "hub.pingcap.net/jenkins/python3-requests:latest" name: "report" resources: limits: memory: "256Mi" cpu: "100m" tty: true volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false - env: - name: "JENKINS_SECRET" value: "********" - name: "JENKINS_TUNNEL" value: "jenkins-agent.apps.svc.cluster.local:50000" - name: "JENKINS_AGENT_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51" - name: "JENKINS_NAME" value: "pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51" - name: "JENKINS_AGENT_WORKDIR" value: "/home/jenkins/agent" - name: "JENKINS_URL" value: "http://jenkins.apps.svc.cluster.local:8080/jenkins/" image: "jenkins/inbound-agent:3206.vb_15dcf73f6a_9-2" name: "jnlp" resources: requests: memory: "256Mi" cpu: "100m" volumeMounts: - mountPath: "/home/jenkins/agent" name: "workspace-volume" readOnly: false restartPolicy: "Never" securityContext: fsGroup: 1000 volumes: - emptyDir: medium: "" name: "workspace-volume" Running on pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51 in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/server_config_compatibility/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:52:54 CST 2024] <<<<<< run test case server_config_compatibility success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_dup_error_restart/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:52:58 CST 2024] <<<<<< run test case changefeed_dup_error_restart success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_big_messages/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:01 CST 2024] <<<<<< run test case kafka_big_messages success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 17.53 secs (212423200 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_compression/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:04 CST 2024] <<<<<< run test case kafka_compression success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_messages/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:07 CST 2024] <<<<<< run test case kafka_messages success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_sink_error_resume/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:11 CST 2024] <<<<<< run test case kafka_sink_error_resume success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 7.16 secs (520167531 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/mq_sink_lost_callback/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:14 CST 2024] <<<<<< run test case mq_sink_lost_callback success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/mq_sink_dispatcher/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:17 CST 2024] <<<<<< run test case mq_sink_dispatcher success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_column_selector/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:20 CST 2024] <<<<<< run test case kafka_column_selector success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_column_selector_avro/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:23 CST 2024] <<<<<< run test case kafka_column_selector_avro success! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/debezium/run.sh using Sink-Type: storage... <<================= find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/lossy_ddl/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/lossy_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 15.43 secs (241308069 bytes/sec) [Pipeline] { [Pipeline] sh [Pipeline] sh [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G04 Run cases: foreign_key ddl_puller_lag ddl_only_block_related_table changefeed_auto_stop PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=750f3243-efa4-4d3c-8c92-b0bffcf859c4 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G04 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-69hgf GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3 pingcap_tiflow_pull_cdc_integration_storage_test_1801-69hgf GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/foreign_key/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G02 [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=1f73b163-84d6-4de8-8b48-b706270c31a0 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G02 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-n3bms GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-n3bms pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:33 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] { [Pipeline] { + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G01 Run cases: http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth kafka_simple_basic kafka_simple_basic_avro kafka_simple_handle_key_only kafka_simple_handle_key_only_avro kafka_simple_claim_check kafka_simple_claim_check_avro canal_json_adapter_compatibility canal_json_basic canal_json_content_compatible multi_topics avro_basic canal_json_handle_key_only open_protocol_handle_key_only canal_json_claim_check open_protocol_claim_check canal_json_storage_basic canal_json_storage_partition_table multi_tables_ddl PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=fd9019c6-4821-429d-9c2c-96736117d7cb BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G01 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-8bm7h GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc pingcap_tiflow_pull_cdc_integration_storage_test_1801-8bm7h GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/http_api/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:33 CST 2024] <<<<<< run test case http_api success! >>>>>> [Pipeline] checkout [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git [Pipeline] checkout The recommended git tool is: git The recommended git tool is: git [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@6b081bff; decorates RemoteLauncher[hudson.remoting.Channel@40344e77:JNLP4-connect connection from 10.233.127.244/10.233.127.244:41174] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Cloning the remote Git repository Using shallow clone with depth 1 Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@670f90c4; decorates RemoteLauncher[hudson.remoting.Channel@5d63bab6:JNLP4-connect connection from 10.233.97.107/10.233.97.107:37540] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@71ea5edb; decorates RemoteLauncher[hudson.remoting.Channel@17c27c7e:JNLP4-connect connection from 10.233.90.75/10.233.90.75:45146] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@7255a23f; decorates RemoteLauncher[hudson.remoting.Channel@20eeb892:JNLP4-connect connection from 10.233.84.76/10.233.84.76:47432] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@419ff5a6; decorates RemoteLauncher[hudson.remoting.Channel@71492115:JNLP4-connect connection from 10.233.100.42/10.233.100.42:42790] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@3023b40d; decorates RemoteLauncher[hudson.remoting.Channel@7ce4a3df:JNLP4-connect connection from 10.233.69.252/10.233.69.252:51584] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@430d7ca; decorates RemoteLauncher[hudson.remoting.Channel@a5f2218:JNLP4-connect connection from 10.233.86.63/10.233.86.63:50414] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow No credentials specified Warning: JENKINS-30600: special launcher org.csanchez.jenkins.plugins.kubernetes.pipeline.ContainerExecDecorator$1@5fe1cce3; decorates RemoteLauncher[hudson.remoting.Channel@3990fb0e:JNLP4-connect connection from 10.233.93.152/10.233.93.152:59782] will be ignored (a typical symptom is the Git executable not being run inside a designated container) Cloning the remote Git repository Using shallow clone with depth 1 [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] { Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 [Pipeline] cache Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 Cloning repository https://github.com/PingCAP-QE/ci.git > git init /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test # timeout=10 Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 start tidb cluster in /tmp/tidb_cdc_test/foreign_key Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Fetching upstream changes from https://github.com/PingCAP-QE/ci.git > git --version # timeout=10 > git --version # 'git version 2.39.2' > git fetch --tags --force --progress --depth=1 -- https://github.com/PingCAP-QE/ci.git +refs/heads/*:refs/remotes/origin/* # timeout=5 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/http_api_tls/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:37 CST 2024] <<<<<< run test case http_api_tls success! >>>>>> > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: storage... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC7EA84F0A26BC < X-Xss-Protection: 1; mode=block < Date: Sun, 05 May 2024 04:53:39 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Commit message: "fix(br): use failpoint tidb-server instead (#2951)" Avoid second fetch Checking out Revision 03312178c534dce949face80c69812d989e55009 (origin/main) Bucket 's3://logbucket/' created [Sun May 5 12:53:39 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>> Exiting on signal: INTERRUPT Commit message: "fix(br): use failpoint tidb-server instead (#2951)" > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 > git config remote.origin.url https://github.com/PingCAP-QE/ci.git # timeout=10 > git config --add remote.origin.fetch +refs/heads/*:refs/remotes/origin/* # timeout=10 > git rev-parse origin/main^{commit} # timeout=10 > git config core.sparsecheckout # timeout=10 > git checkout -f 03312178c534dce949face80c69812d989e55009 # timeout=10 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/api_v2/run.sh using Sink-Type: storage... <<================= Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c67046c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp, pid:2372, start at 2024-05-05 12:53:41.310396344 +0800 CST m=+5.348202194 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:55:41.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:53:41.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:43:41.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c67046c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp, pid:2372, start at 2024-05-05 12:53:41.310396344 +0800 CST m=+5.348202194 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:55:41.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:53:41.325 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:43:41.325 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6705e00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp, pid:2457, start at 2024-05-05 12:53:41.392585642 +0800 CST m=+5.373915126 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:55:41.402 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:53:41.368 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:43:41.368 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/lossy_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/lossy_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/lossy_ddl/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/lossy_ddl/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/lossy_ddl/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 8.30 secs (448617193 bytes/sec) [Pipeline] { [Pipeline] cache [Sun May 5 12:53:46 CST 2024] <<<<<< START cdc server in lossy_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.lossy_ddl.38173819.out server --log-file /tmp/tidb_cdc_test/lossy_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/lossy_ddl/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/http_api_tls_with_user_auth/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:45 CST 2024] <<<<<< run test case http_api_tls_with_user_auth success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c675fac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:1244, start at 2024-05-05 12:53:47.152223765 +0800 CST m=+5.255549229 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:55:47.158 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:53:47.115 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:43:47.115 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:53:49 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2af076bd-e37b-4b65-9097-0208afb1fcc9 {"id":"2af076bd-e37b-4b65-9097-0208afb1fcc9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884826} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4719948fca 2af076bd-e37b-4b65-9097-0208afb1fcc9 /tidb/cdc/default/default/upstream/7365374186210334436 {"id":7365374186210334436,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2af076bd-e37b-4b65-9097-0208afb1fcc9 {"id":"2af076bd-e37b-4b65-9097-0208afb1fcc9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884826} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4719948fca 2af076bd-e37b-4b65-9097-0208afb1fcc9 /tidb/cdc/default/default/upstream/7365374186210334436 {"id":7365374186210334436,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2af076bd-e37b-4b65-9097-0208afb1fcc9 {"id":"2af076bd-e37b-4b65-9097-0208afb1fcc9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884826} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4719948fca 2af076bd-e37b-4b65-9097-0208afb1fcc9 /tidb/cdc/default/default/upstream/7365374186210334436 {"id":7365374186210334436,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: f5fb6264-e1c6-4cec-870e-92024bbe5d56 Info: {"upstream_id":7365374186210334436,"namespace":"default","id":"f5fb6264-e1c6-4cec-870e-92024bbe5d56","sink_uri":"blackhole:","create_time":"2024-05-05T12:53:49.530176335+08:00","start_ts":449546768724787205,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546768724787205,"checkpoint_ts":449546768724787205,"checkpoint_time":"2024-05-05 12:53:49.425"} =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:49 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cli_tls_with_auth/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c675fac0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:1244, start at 2024-05-05 12:53:47.152223765 +0800 CST m=+5.255549229 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:55:47.158 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:53:47.115 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:43:47.115 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c67611c0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:1328, start at 2024-05-05 12:53:47.222794134 +0800 CST m=+5.257984783 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:55:47.228 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:53:47.207 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:43:47.207 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/foreign_key/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/foreign_key/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/foreign_key/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2652.out cli tso query --pd=http://127.0.0.1:2379 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:52 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x + tso='449546769534812161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546769534812161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 12:53:54 CST 2024] <<<<<< START cdc server in foreign_key case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.26902692.out server --log-file /tmp/tidb_cdc_test/foreign_key/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/foreign_key/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:53:56 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:53:57 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3107da6-13d8-4d70-9780-e02371311caf {"id":"d3107da6-13d8-4d70-9780-e02371311caf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4719adb5d5 d3107da6-13d8-4d70-9780-e02371311caf /tidb/cdc/default/default/upstream/7365374211930046388 {"id":7365374211930046388,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3107da6-13d8-4d70-9780-e02371311caf {"id":"d3107da6-13d8-4d70-9780-e02371311caf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4719adb5d5 d3107da6-13d8-4d70-9780-e02371311caf /tidb/cdc/default/default/upstream/7365374211930046388 {"id":7365374211930046388,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d3107da6-13d8-4d70-9780-e02371311caf {"id":"d3107da6-13d8-4d70-9780-e02371311caf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884834} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4719adb5d5 d3107da6-13d8-4d70-9780-e02371311caf /tidb/cdc/default/default/upstream/7365374211930046388 {"id":7365374211930046388,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.foreign_key.cli.2754.out cli changefeed create --start-ts=449546769534812161 '--sink-uri=file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-8845?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 5b577e06-5c0f-40b6-96ac-ba8c5907aa0a Info: {"upstream_id":7365374211930046388,"namespace":"default","id":"5b577e06-5c0f-40b6-96ac-ba8c5907aa0a","sink_uri":"file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-8845?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:53:57.526162374+08:00","start_ts":449546769534812161,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546769534812161,"checkpoint_ts":449546769534812161,"checkpoint_time":"2024-05-05 12:53:52.515"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 1-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x + workdir=/tmp/tidb_cdc_test/foreign_key + sink_uri='file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-8845?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/foreign_key ++ date + echo '[Sun May 5 12:53:58 CST 2024] <<<<<< START storage consumer in foreign_key case >>>>>>' [Sun May 5 12:53:58 CST 2024] <<<<<< START storage consumer in foreign_key case >>>>>> + cd /tmp/tidb_cdc_test/foreign_key + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/foreign_key + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/foreign_key/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/foreign_key/storage_test/ticdc-foreign-key-test-8845?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 2-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 11.56 secs (322112431 bytes/sec) [Pipeline] { [Pipeline] cache Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: storage... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide table foreign_key.finish_mark not exists for 2-th check, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 3-th time, retry later * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC7EAE08A41C43 < X-Xss-Protection: 1; mode=block < Date: Sun, 05 May 2024 04:54:04 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Sun May 5 12:54:04 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Exiting on signal: INTERRUPT ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 4-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:54:07 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c688cd80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:1655, start at 2024-05-05 12:54:06.429118666 +0800 CST m=+6.127559769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:56:06.438 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:54:06.440 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:44:06.440 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c688cd80019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:1655, start at 2024-05-05 12:54:06.429118666 +0800 CST m=+6.127559769 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:56:06.438 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:54:06.440 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:44:06.440 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c68802c0006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:1739, start at 2024-05-05 12:54:05.582541586 +0800 CST m=+5.180214008 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:56:05.589 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:54:05.579 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:44:05.579 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_tls_with_auth/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table foreign_key.finish_mark not exists for 5-th check, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 4-th time, retry later table foreign_key.finish_mark not exists for 6-th check, retry later The 1 times to try to start tls tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/cli_tls_with_auth Starting TLS PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting TLS TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table foreign_key.finish_mark not exists for 7-th check, retry later Starting TLS TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying TLS TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:54:13 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>> table foreign_key.finish_mark not exists for 8-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table foreign_key.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 5-th time, retry later find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:54:18 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>> table foreign_key.finish_mark not exists for 10-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6953800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:3346, start at 2024-05-05 12:54:19.126222122 +0800 CST m=+5.126535166 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:56:19.133 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:54:19.104 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:44:19.104 +0800 All versions after safe point can be accessed. (DO NOT EDIT) + pd_host=127.0.0.1 + pd_port=2579 + is_tls=true + '[' true == true ']' ++ run_cdc_cli tso query --pd=https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3413.out cli tso query --pd=https://127.0.0.1:2579 table foreign_key.finish_mark not exists for 11-th check, retry later + set +x + tso='449546776871960577 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546776871960577 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 12:54:22 CST 2024] <<<<<< START cdc server in cli_tls_with_auth case >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:54:21 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates ']' + curl_status_cmd='curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.34673469.out server --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_cli_tls_with_auth_tls1.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_data_cli_tls_with_auth_tls1 --cluster-id default --config /tmp/tidb_cdc_test/cli_tls_with_auth/server.toml --ca /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server-key.pem --cert-allowed-cn client --addr 127.0.0.1:8300 --pd https://127.0.0.1:2579 + (( i = 0 )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table foreign_key.finish_mark not exists for 12-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 21.39 secs (174107719 bytes/sec) [Pipeline] { [Pipeline] cache find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup + CDC_BINARY=cdc.test + SINK_TYPE=storage + EXIST_FILES=() + CLEANED_FILES=() + trap stop_tidb_cluster EXIT + run storage + '[' storage '!=' storage ']' + rm -rf /tmp/tidb_cdc_test/storage_cleanup + mkdir -p /tmp/tidb_cdc_test/storage_cleanup + start_tidb_cluster --workdir /tmp/tidb_cdc_test/storage_cleanup The 1 times to try to start tidb cluster... table foreign_key.finish_mark not exists for 13-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl --cacert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --user ticdc:ticdc_secret -vsL --max-time 20 https://127.0.0.1:8300/debug/info * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Initializing NSS with certpath: sql:/etc/pki/nssdb * CAfile: /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem CApath: none * NSS: client certificate from file * subject: CN=client * start date: Feb 18 07:48:00 2020 GMT * expire date: Jan 25 07:48:00 2120 GMT * common name: client * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * SSL connection using TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 * Server certificate: * subject: CN=tidb-server * start date: Feb 18 09:11:00 2020 GMT * expire date: Jan 25 09:11:00 2120 GMT * common name: tidb-server * issuer: CN=My own CA,O=PingCAP,L=Beijing,ST=Beijing,C=CN * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:54:25 GMT < Content-Length: 1241 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e8e3f85-41d6-4dcb-9336-8e3250ac5367 {"id":"5e8e3f85-41d6-4dcb-9336-8e3250ac5367","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884862} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f471a343582 5e8e3f85-41d6-4dcb-9336-8e3250ac5367 /tidb/cdc/default/default/upstream/7365374355680606051 {"id":7365374355680606051,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e8e3f85-41d6-4dcb-9336-8e3250ac5367 {"id":"5e8e3f85-41d6-4dcb-9336-8e3250ac5367","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884862} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f471a343582 5e8e3f85-41d6-4dcb-9336-8e3250ac5367 /tidb/cdc/default/default/upstream/7365374355680606051 {"id":7365374355680606051,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e8e3f85-41d6-4dcb-9336-8e3250ac5367 {"id":"5e8e3f85-41d6-4dcb-9336-8e3250ac5367","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884862} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/1f5d8f471a343582 5e8e3f85-41d6-4dcb-9336-8e3250ac5367 /tidb/cdc/default/default/upstream/7365374355680606051 {"id":7365374355680606051,"pd-endpoints":"https://127.0.0.1:2579,https://127.0.0.1:2579","key-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server-key.pem","cert-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/server.pem","ca-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem","cert-allowed-cn":["client","tidb-server"]}' + grep -q 'etcd info' + break + set +x table foreign_key.finish_mark not exists for 14-th check, retry later check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl can't found finish mark run task failed 6-th time, retry later + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3535.out cli changefeed create --start-ts=449546776871960577 '--sink-uri=file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-20058?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365374355680606051,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-20058?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:54:28.77414924+08:00","start_ts":449546776871960577,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546776871960577,"checkpoint_ts":449546776871960577,"checkpoint_time":"2024-05-05 12:54:20.504"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table foreign_key.finish_mark not exists for 15-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/cli_tls_with_auth + sink_uri='file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-20058?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/cli_tls_with_auth ++ date + echo '[Sun May 5 12:54:30 CST 2024] <<<<<< START storage consumer in cli_tls_with_auth case >>>>>>' [Sun May 5 12:54:30 CST 2024] <<<<<< START storage consumer in cli_tls_with_auth case >>>>>> + cd /tmp/tidb_cdc_test/cli_tls_with_auth + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/cli_tls_with_auth + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/cli_tls_with_auth/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-20058?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table test.simple not exists for 1-th check, retry later table foreign_key.finish_mark not exists for 16-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/storage_cleanup Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table test.simple not exists for 2-th check, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 6.01 secs (619277772 bytes/sec) [Pipeline] { [Pipeline] cache table foreign_key.finish_mark not exists for 17-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.simple not exists for 3-th check, retry later table foreign_key.finish_mark not exists for 18-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple not exists for 4-th check, retry later table foreign_key.finish_mark not exists for 19-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table test.simple not exists for 5-th check, retry later table foreign_key.finish_mark not exists for 20-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_lossy_ddl /tmp/tidb_cdc_test/lossy_ddl [2024/05/05 12:53:53.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546768790323216,\"CommitTs\":449546768790323217,\"RowID\":1,\"PhysicalTableID\":106,\"TableInfo\":{\"id\":106,\"name\":{\"O\":\"example1\",\"L\":\"example1\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546768777478177,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example1\",\"TableID\":106,\"IsPartition\":false},\"Version\":449546768777478183,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449546769694720016,\"HandleKey\":1}"] [2024/05/05 12:53:55.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546768895180810,\"CommitTs\":449546768895180812,\"RowID\":1,\"PhysicalTableID\":109,\"TableInfo\":{\"id\":109,\"name\":{\"O\":\"example2\",\"L\":\"example2\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546768868966435,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example2\",\"TableID\":109,\"IsPartition\":false},\"Version\":449546768868966439,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449546770219008007,\"HandleKey\":1}"] [2024/05/05 12:53:57.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546768986931213,\"CommitTs\":449546768986931216,\"RowID\":1,\"PhysicalTableID\":112,\"TableInfo\":{\"id\":112,\"name\":{\"O\":\"example3\",\"L\":\"example3\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":7,\"Flag\":128,\"Flen\":19,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546768973824003,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example3\",\"TableID\":112,\"IsPartition\":false},\"Version\":449546768973824007,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"2023-04-19 11:48:00\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":36,\"SplitTxn\":false,\"ReplicatingTs\":449546770743296040,\"HandleKey\":1}"] [2024/05/05 12:53:59.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769078681614,\"CommitTs\":449546769078681616,\"RowID\":1,\"PhysicalTableID\":115,\"TableInfo\":{\"id\":115,\"name\":{\"O\":\"example4\",\"L\":\"example4\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769065574410,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example4\",\"TableID\":115,\"IsPartition\":false},\"Version\":449546769065574414,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449546771280691204,\"HandleKey\":1}"] [2024/05/05 12:54:01.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769170432019,\"CommitTs\":449546769170432021,\"RowID\":1,\"PhysicalTableID\":118,\"TableInfo\":{\"id\":118,\"name\":{\"O\":\"example5\",\"L\":\"example5\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769157324824,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example5\",\"TableID\":118,\"IsPartition\":false},\"Version\":449546769157324829,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449546771791872007,\"HandleKey\":1}"] [2024/05/05 12:54:03.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769262182409,\"CommitTs\":449546769262182411,\"RowID\":1,\"PhysicalTableID\":121,\"TableInfo\":{\"id\":121,\"name\":{\"O\":\"example6\",\"L\":\"example6\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769235968032,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example6\",\"TableID\":121,\"IsPartition\":false},\"Version\":449546769235968036,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449546772316160008,\"HandleKey\":1}"] [2024/05/05 12:54:05.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769340825616,\"CommitTs\":449546769340825618,\"RowID\":1,\"PhysicalTableID\":124,\"TableInfo\":{\"id\":124,\"name\":{\"O\":\"example7\",\"L\":\"example7\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8\",\"Collate\":\"utf8_general_ci\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769327718429,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example7\",\"TableID\":124,\"IsPartition\":false},\"Version\":449546769327718434,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449546772853555204,\"HandleKey\":1}"] [2024/05/05 12:54:07.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769380147216,\"CommitTs\":449546769380147218,\"RowID\":-1,\"PhysicalTableID\":128,\"TableInfo\":{\"id\":127,\"name\":{\"O\":\"example8\",\"L\":\"example8\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769367040032,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":128,\"name\":{\"O\":\"b0\",\"L\":\"b0\"},\"less_than\":[\"0\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":129,\"name\":{\"O\":\"b1\",\"L\":\"b1\"},\"less_than\":[\"MAXVALUE\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":2,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example8\",\"TableID\":127,\"IsPartition\":true},\"Version\":449546769367040037,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":-1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449546773364736007,\"HandleKey\":-1}"] [2024/05/05 12:54:07.246 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769380147219,\"CommitTs\":449546769380147220,\"RowID\":1,\"PhysicalTableID\":129,\"TableInfo\":{\"id\":127,\"name\":{\"O\":\"example8\",\"L\":\"example8\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769367040032,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":128,\"name\":{\"O\":\"b0\",\"L\":\"b0\"},\"less_than\":[\"0\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":129,\"name\":{\"O\":\"b1\",\"L\":\"b1\"},\"less_than\":[\"MAXVALUE\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":2,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example8\",\"TableID\":127,\"IsPartition\":true},\"Version\":449546769367040037,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449546773364736008,\"HandleKey\":1}"] [2024/05/05 12:54:09.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769458790414,\"CommitTs\":449546769458790415,\"RowID\":1,\"PhysicalTableID\":132,\"TableInfo\":{\"id\":132,\"name\":{\"O\":\"example9\",\"L\":\"example9\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769445683207,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example9\",\"TableID\":132,\"IsPartition\":false},\"Version\":449546769445683212,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449546773889024008,\"HandleKey\":1}"] [2024/05/05 12:54:11.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769498112014,\"CommitTs\":449546769498112015,\"RowID\":1,\"PhysicalTableID\":135,\"TableInfo\":{\"id\":135,\"name\":{\"O\":\"example10\",\"L\":\"example10\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":5,\"Flag\":0,\"Flen\":22,\"Decimal\":-1,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769485004808,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example10\",\"TableID\":135,\"IsPartition\":false},\"Version\":449546769485004812,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":36,\"SplitTxn\":false,\"ReplicatingTs\":449546774413312006,\"HandleKey\":1}"] [2024/05/05 12:54:13.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769589862416,\"CommitTs\":449546769589862418,\"RowID\":1,\"PhysicalTableID\":138,\"TableInfo\":{\"id\":138,\"name\":{\"O\":\"example11\",\"L\":\"example11\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":8,\"Flag\":4099,\"Flen\":20,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":8,\"Flag\":0,\"Flen\":20,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769576755222,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example11\",\"TableID\":138,\"IsPartition\":false},\"Version\":449546769576755227,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449546774937600007,\"HandleKey\":1}"] [2024/05/05 12:54:15.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769681612816,\"CommitTs\":449546769681612818,\"RowID\":1,\"PhysicalTableID\":141,\"TableInfo\":{\"id\":141,\"name\":{\"O\":\"example12\",\"L\":\"example12\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":251,\"Flag\":0,\"Flen\":4294967295,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769668505630,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example12\",\"TableID\":141,\"IsPartition\":false},\"Version\":449546769668505635,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449546775461888008,\"HandleKey\":1}"] [2024/05/05 12:54:17.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769773363216,\"CommitTs\":449546769773363219,\"RowID\":1,\"PhysicalTableID\":144,\"TableInfo\":{\"id\":144,\"name\":{\"O\":\"example13\",\"L\":\"example13\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":247,\"Flag\":0,\"Flen\":1,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":[\"a\",\"b\",\"c\"],\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769760256032,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example13\",\"TableID\":144,\"IsPartition\":false},\"Version\":449546769760256036,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449546775999283204,\"HandleKey\":1}"] [2024/05/05 12:54:19.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769865113617,\"CommitTs\":449546769878220802,\"RowID\":1,\"PhysicalTableID\":147,\"TableInfo\":{\"id\":147,\"name\":{\"O\":\"example14\",\"L\":\"example14\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":15,\"Flag\":0,\"Flen\":256,\"Decimal\":0,\"Charset\":\"utf8mb4\",\"Collate\":\"utf8mb4_bin\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769852006433,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example14\",\"TableID\":147,\"IsPartition\":false},\"Version\":449546769852006438,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":64},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":\"MjAyMy0wNC0xOSAxMTo0ODowMA==\"}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":47,\"SplitTxn\":false,\"ReplicatingTs\":449546776523571204,\"HandleKey\":1}"] [2024/05/05 12:54:21.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546769969971220,\"CommitTs\":449546769969971222,\"RowID\":1,\"PhysicalTableID\":150,\"TableInfo\":{\"id\":150,\"name\":{\"O\":\"example15\",\"L\":\"example15\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2},{\"id\":2,\"name\":{\"O\":\"b\",\"L\":\"b\"},\"offset\":1,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":0,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":2,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546769956864002,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example15\",\"TableID\":150,\"IsPartition\":false},\"Version\":449546769956864006,\"RowColumnsOffset\":{\"1\":0,\"2\":1},\"ColumnsFlag\":{\"1\":11,\"2\":65},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1},{\"column_id\":2,\"value\":-1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":29,\"SplitTxn\":false,\"ReplicatingTs\":449546777047859204,\"HandleKey\":1}"] [2024/05/05 12:54:23.246 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546770061721615,\"CommitTs\":449546770061721617,\"RowID\":1,\"PhysicalTableID\":154,\"TableInfo\":{\"id\":153,\"name\":{\"O\":\"example16\",\"L\":\"example16\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546770048614409,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`a`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":154,\"name\":{\"O\":\"p0\",\"L\":\"p0\"},\"less_than\":[\"6\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":155,\"name\":{\"O\":\"p1\",\"L\":\"p1\"},\"less_than\":[\"11\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":156,\"name\":{\"O\":\"p2\",\"L\":\"p2\"},\"less_than\":[\"21\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":3,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example16\",\"TableID\":153,\"IsPartition\":true},\"Version\":449546770048614413,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449546777572409348,\"HandleKey\":1}"] [2024/05/05 12:54:25.246 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546770074828817,\"CommitTs\":449546770074828818,\"RowID\":18,\"PhysicalTableID\":158,\"TableInfo\":{\"id\":158,\"name\":{\"O\":\"example17\",\"L\":\"example17\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546770061721635,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example17\",\"TableID\":158,\"IsPartition\":false},\"Version\":449546770061721639,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":18}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449546778096435204,\"HandleKey\":18}"] [2024/05/05 12:54:27.246 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546770140364813,\"CommitTs\":449546770140364815,\"RowID\":1,\"PhysicalTableID\":162,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546770114150434,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":162,\"name\":{\"O\":\"pBefore1950\",\"L\":\"pbefore1950\"},\"less_than\":[\"1950\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":163,\"name\":{\"O\":\"p1950\",\"L\":\"p1950\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":166,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":167,\"name\":{\"O\":\"p1990\",\"L\":\"p1990\"},\"less_than\":[\"2000\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":6,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449546770127257601,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449546778620985348,\"HandleKey\":1}"] [2024/05/05 12:54:27.246 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546770140364813,\"CommitTs\":449546770140364815,\"RowID\":1977,\"PhysicalTableID\":165,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546770114150434,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":162,\"name\":{\"O\":\"pBefore1950\",\"L\":\"pbefore1950\"},\"less_than\":[\"1950\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":163,\"name\":{\"O\":\"p1950\",\"L\":\"p1950\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":166,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":167,\"name\":{\"O\":\"p1990\",\"L\":\"p1990\"},\"less_than\":[\"2000\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":6,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449546770127257601,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1977}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449546778620985352,\"HandleKey\":1977}"] [2024/05/05 12:54:27.246 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546770140364813,\"CommitTs\":449546770140364815,\"RowID\":1999,\"PhysicalTableID\":167,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546770114150434,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":[],\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":162,\"name\":{\"O\":\"pBefore1950\",\"L\":\"pbefore1950\"},\"less_than\":[\"1950\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":163,\"name\":{\"O\":\"p1950\",\"L\":\"p1950\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":166,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":167,\"name\":{\"O\":\"p1990\",\"L\":\"p1990\"},\"less_than\":[\"2000\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":[],\"dropping_definitions\":[],\"NewPartitionIDs\":null,\"states\":null,\"num\":6,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":0,\"ddl_expr\":\"\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449546770127257601,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1999}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449546778620985354,\"HandleKey\":1999}"] [2024/05/05 12:54:33.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546770336972811,\"CommitTs\":449546770336972812,\"RowID\":2003,\"PhysicalTableID\":174,\"TableInfo\":{\"id\":161,\"name\":{\"O\":\"example18\",\"L\":\"example18\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"id\",\"L\":\"id\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":null,\"constraint_info\":null,\"fk_info\":null,\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546770310758409,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":{\"type\":1,\"expr\":\"`id`\",\"columns\":null,\"enable\":true,\"is_empty_columns\":false,\"definitions\":[{\"id\":169,\"name\":{\"O\":\"pBefore1960\",\"L\":\"pbefore1960\"},\"less_than\":[\"1960\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":164,\"name\":{\"O\":\"p1960\",\"L\":\"p1960\"},\"less_than\":[\"1970\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":165,\"name\":{\"O\":\"p1970\",\"L\":\"p1970\"},\"less_than\":[\"1980\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":172,\"name\":{\"O\":\"p1980\",\"L\":\"p1980\"},\"less_than\":[\"1990\"],\"in_values\":null,\"policy_ref_info\":null},{\"id\":174,\"name\":{\"O\":\"p1990to2010\",\"L\":\"p1990to2010\"},\"less_than\":[\"2010\"],\"in_values\":null,\"policy_ref_info\":null}],\"adding_definitions\":null,\"dropping_definitions\":null,\"NewPartitionIDs\":null,\"states\":null,\"num\":5,\"ddl_state\":0,\"new_table_id\":0,\"ddl_type\":1,\"ddl_expr\":\"`id`\",\"ddl_columns\":null},\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example18\",\"TableID\":161,\"IsPartition\":true},\"Version\":449546770310758414,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":2003}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449546780193587204,\"HandleKey\":2003}"] [2024/05/05 12:54:37.245 +08:00] [DEBUG] [black_hole_dml_sink.go:45] ["BlackHoleSink: WriteEvents"] [row="{\"StartTs\":449546770756403225,\"CommitTs\":449546770756403226,\"RowID\":1,\"PhysicalTableID\":182,\"TableInfo\":{\"id\":182,\"name\":{\"O\":\"example19\",\"L\":\"example19\"},\"charset\":\"utf8mb4\",\"collate\":\"utf8mb4_bin\",\"cols\":[{\"id\":1,\"name\":{\"O\":\"a\",\"L\":\"a\"},\"offset\":0,\"origin_default\":null,\"origin_default_bit\":null,\"default\":null,\"default_bit\":null,\"default_is_expr\":false,\"generated_expr_string\":\"\",\"generated_stored\":false,\"dependences\":null,\"type\":{\"Tp\":3,\"Flag\":4099,\"Flen\":11,\"Decimal\":0,\"Charset\":\"binary\",\"Collate\":\"binary\",\"Elems\":null,\"ElemsIsBinaryLit\":null,\"Array\":false},\"state\":5,\"comment\":\"\",\"hidden\":false,\"change_state_info\":null,\"version\":2}],\"index_info\":[],\"constraint_info\":null,\"fk_info\":[],\"state\":5,\"pk_is_handle\":true,\"is_common_handle\":false,\"common_handle_version\":0,\"comment\":\"\",\"auto_inc_id\":0,\"auto_id_cache\":0,\"auto_rand_id\":0,\"max_col_id\":1,\"max_idx_id\":0,\"max_fk_id\":0,\"max_cst_id\":0,\"update_timestamp\":449546770743296022,\"ShardRowIDBits\":0,\"max_shard_row_id_bits\":0,\"auto_random_bits\":0,\"auto_random_range_bits\":0,\"pre_split_regions\":0,\"partition\":null,\"compression\":\"\",\"view\":null,\"sequence\":null,\"Lock\":null,\"version\":5,\"tiflash_replica\":null,\"is_columnar\":false,\"temp_table_type\":0,\"cache_table_status\":0,\"policy_ref_info\":null,\"stats_options\":null,\"exchange_partition_info\":null,\"ttl_info\":null,\"SchemaID\":104,\"TableName\":{\"Schema\":\"lossy_ddl\",\"Table\":\"example19\",\"TableID\":182,\"IsPartition\":false},\"Version\":449546770743296027,\"RowColumnsOffset\":{\"1\":0},\"ColumnsFlag\":{\"1\":11},\"HandleIndexID\":-1,\"IndexColumnsOffset\":[[0]]},\"Columns\":[{\"column_id\":1,\"value\":1}],\"PreColumns\":null,\"Checksum\":null,\"ApproximateDataSize\":25,\"SplitTxn\":false,\"ReplicatingTs\":449546781242163204,\"HandleKey\":1}"] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table test.simple exists table test.`simple-dash` exists + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546781996875781, "checkpoint_time": "2024-05-05 12:54:40.054", "error": null }' + echo '{ "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546781996875781, "checkpoint_time": "2024-05-05 12:54:40.054", "error": null }' { "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546781996875781, "checkpoint_time": "2024-05-05 12:54:40.054", "error": null } ++ echo '{' '"upstream_id":' 7365374355680606051, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546781996875781, '"checkpoint_time":' '"2024-05-05' '12:54:40.054",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365374355680606051, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546781996875781, '"checkpoint_time":' '"2024-05-05' '12:54:40.054",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:54:40 CST 2024] <<<<<< run test case lossy_ddl success! >>>>>> table foreign_key.finish_mark exists check diff successfully changefeed count 1 check pass, pd_addr: https://127.0.0.1:2579 wait process cdc.test exit for 1-th time... Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc00147d230} {Type:update Path:[Config SyncPointRetention] From: To:0xc00147d238} {Type:update Path:[Config Consistent] From: To:0xc001265ea0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3802.out cli changefeed --changefeed-id custom-changefeed-name pause wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:54:43 CST 2024] <<<<<< run test case foreign_key success! >>>>>> + set +x VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6abac00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:2191, start at 2024-05-05 12:54:42.139339987 +0800 CST m=+6.595801353 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:56:42.145 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:54:42.146 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:44:42.146 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6abac00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:2191, start at 2024-05-05 12:54:42.139339987 +0800 CST m=+6.595801353 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:56:42.145 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:54:42.146 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:44:42.146 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6aa4080014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:2268, start at 2024-05-05 12:54:40.673889152 +0800 CST m=+5.050959000 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:56:40.680 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:54:40.642 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:44:40.642 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/storage_cleanup/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/storage_cleanup/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/storage_cleanup/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/storage_cleanup/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/storage_cleanup/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 10.08 secs (369426594 bytes/sec) [Pipeline] { [Pipeline] cache + cd /tmp/tidb_cdc_test/storage_cleanup + run_cdc_server --workdir /tmp/tidb_cdc_test/storage_cleanup --binary cdc.test [Sun May 5 12:54:46 CST 2024] <<<<<< START cdc server in storage_cleanup case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_cleanup.36483650.out server --log-file /tmp/tidb_cdc_test/storage_cleanup/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/storage_cleanup/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449546782521425925, "checkpoint_time": "2024-05-05 12:54:42.055", "error": null }' + echo '{ "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449546782521425925, "checkpoint_time": "2024-05-05 12:54:42.055", "error": null }' { "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449546782521425925, "checkpoint_time": "2024-05-05 12:54:42.055", "error": null } ++ echo '{' '"upstream_id":' 7365374355680606051, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449546782521425925, '"checkpoint_time":' '"2024-05-05' '12:54:42.055",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365374355680606051, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449546782521425925, '"checkpoint_time":' '"2024-05-05' '12:54:42.055",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3876.out cli changefeed update --pd=https://127.0.0.1:2579 --config=/tmp/tidb_cdc_test/cli_tls_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc00199ddc8} {Type:update Path:[Config SyncPointRetention] From: To:0xc00199ddd8} {Type:update Path:[Config Consistent] From: To:0xc0013789a0} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365374355680606051,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-20058?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:54:28.77414924+08:00","start_ts":449546776871960577,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":0,"checkpoint_ts":449546782521425925,"checkpoint_time":"2024-05-05 12:54:42.055"} PASS coverage: 2.8% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:54:49 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6403a4fa-ff30-47d8-b89f-16687dcc2d09 {"id":"6403a4fa-ff30-47d8-b89f-16687dcc2d09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884886} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471a831cd0 6403a4fa-ff30-47d8-b89f-16687dcc2d09 /tidb/cdc/default/default/upstream/7365374442769582770 {"id":7365374442769582770,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6403a4fa-ff30-47d8-b89f-16687dcc2d09 {"id":"6403a4fa-ff30-47d8-b89f-16687dcc2d09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884886} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471a831cd0 6403a4fa-ff30-47d8-b89f-16687dcc2d09 /tidb/cdc/default/default/upstream/7365374442769582770 {"id":7365374442769582770,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6403a4fa-ff30-47d8-b89f-16687dcc2d09 {"id":"6403a4fa-ff30-47d8-b89f-16687dcc2d09","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884886} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471a831cd0 6403a4fa-ff30-47d8-b89f-16687dcc2d09 /tidb/cdc/default/default/upstream/7365374442769582770 {"id":7365374442769582770,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + generate_historic_files + local target_bucket=storage_test ++ date -d yesterday +%Y-%m-%d + yesterday=2024-05-04 ++ date -d '2 days ago' +%Y-%m-%d + day_before_yesterday=2024-05-03 + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_data_type 2024-05-04 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_data_type + local day=2024-05-04 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_charset 2024-05-03 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_charset + local day=2024-05-03 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test binary_columns 2024-05-03 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=binary_columns + local day=2024-05-03 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_data_type_dummy 2024-05-03 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_data_type_dummy + local day=2024-05-03 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test multi_charset_dummy 2024-05-03 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=multi_charset_dummy + local day=2024-05-03 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test binary_columns_dummy 2024-05-04 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test + local table=binary_columns_dummy + local day=2024-05-04 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test2 multi_data_type 2024-05-03 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test2 + local table=multi_data_type + local day=2024-05-03 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test2 multi_charset 2024-05-03 10 true + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test2 + local table=multi_charset + local day=2024-05-03 + local file_cnt=10 + local should_clean=true + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/1.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/2.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/3.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/4.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/5.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/6.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/7.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/8.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/9.data + '[' true == true ']' + CLEANED_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/10.data + '[' true == true ']' + CLEANED_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test test2 binary_columns 2024-05-04 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test + local schema=test2 + local table=binary_columns + local day=2024-05-04 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test_default test multi_data_type 2022-01-01 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test_default + local schema=test + local table=multi_data_type + local day=2022-01-01 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test_default test multi_charset 2022-01-02 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test_default + local schema=test + local table=multi_charset + local day=2022-01-02 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/meta/CDC.index + generate_single_table_files /tmp/tidb_cdc_test/storage_cleanup storage_test_default test binary_columns 2022-01-03 10 false + local workdir=/tmp/tidb_cdc_test/storage_cleanup + local bucket=storage_test_default + local schema=test + local table=binary_columns + local day=2022-01-03 + local file_cnt=10 + local should_clean=false + table_dir=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03 + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03 ++ seq 1 10 + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data + '[' false == true ']' + EXIST_FILES+=($file) + for i in '$(seq 1 $file_cnt)' + file=/tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data + '[' false == true ']' + EXIST_FILES+=($file) + mkdir -p /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/meta + touch /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/meta/CDC.index + SINK_URI_DEFAULT='file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s' + run_cdc_cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s' -c default-config-test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed-default.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_cleanup.cli.3881.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s' -c default-config-test --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed-default.toml + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.3914.out cli changefeed --changefeed-id custom-changefeed-name resume Create changefeed successfully! ID: default-config-test Info: {"upstream_id":7365374442769582770,"namespace":"default","id":"default-config-test","sink_uri":"file:///tmp/tidb_cdc_test/storage_cleanup/storage_test_default?flush-interval=5s","create_time":"2024-05-05T12:54:49.873730611+08:00","start_ts":449546784550944771,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546784550944771,"checkpoint_ts":449546784550944771,"checkpoint_time":"2024-05-05 12:54:49.797"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + set +x + sleep 20 + set +x start tidb cluster in /tmp/tidb_cdc_test/storage_csv_update Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_puller_lag/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546786204286980, "checkpoint_time": "2024-05-05 12:54:56.104", "error": null }' + echo '{ "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546786204286980, "checkpoint_time": "2024-05-05 12:54:56.104", "error": null }' { "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546786204286980, "checkpoint_time": "2024-05-05 12:54:56.104", "error": null } ++ echo '{' '"upstream_id":' 7365374355680606051, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546786204286980, '"checkpoint_time":' '"2024-05-05' '12:54:56.104",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365374355680606051, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546786204286980, '"checkpoint_time":' '"2024-05-05' '12:54:56.104",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4007.out cli changefeed --changefeed-id custom-changefeed-name remove start tidb cluster in /tmp/tidb_cdc_test/ddl_puller_lag Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449546786466430980 SinkURI: file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-20058?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release changefeed count 0 check pass, pd_addr: https://127.0.0.1:2579 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4094.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-20058?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365374355680606051,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_tls_with_auth/storage_test/ticdc-cli-test-20058?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:55:03.172648429+08:00","start_ts":449546788013080579,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546788013080579,"checkpoint_ts":449546788013080579,"checkpoint_time":"2024-05-05 12:55:03.004"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 16.01 secs (232521450 bytes/sec) [Pipeline] { [Pipeline] sh VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6c0f14001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp, pid:5021, start at 2024-05-05 12:55:03.907624923 +0800 CST m=+5.199168070 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:03.914 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:03.877 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:03.877 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6c0f14001b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp, pid:5021, start at 2024-05-05 12:55:03.907624923 +0800 CST m=+5.199168070 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:03.914 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:03.877 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:03.877 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6c0fc00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-spl-vqwsp, pid:5106, start at 2024-05-05 12:55:03.939827715 +0800 CST m=+5.164781590 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:03.948 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:03.920 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:03.920 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/storage_csv_update/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/storage_csv_update/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/storage_csv_update/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/storage_csv_update/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G03 [Pipeline] sh Run cases: row_format drop_many_tables processor_stop_delay partition_table PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=93d8cc86-d917-4403-9e37-f65dd5fceb1e BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G03 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-r7gz8 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-r7gz8 pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/row_format/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G05 Run cases: charset_gbk ddl_manager multi_source PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=2f1d1f84-bbee-4ae2-bfe9-165993deefae BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G05 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-20f8d GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-20f8d pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/charset_gbk/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G06 Run cases: sink_retry changefeed_error ddl_sequence resourcecontrol PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=20b6f1a7-5504-4a29-8f12-904410223e79 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G06 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-xj7hn GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc pingcap_tiflow_pull_cdc_integration_storage_test_1801-xj7hn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sink_retry/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G07 Run cases: kv_client_stream_reconnect cdc split_region PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=cc518249-0191-4116-8ec6-27202e8e7ef6 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G07 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-7f60r GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp pingcap_tiflow_pull_cdc_integration_storage_test_1801-7f60r GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kv_client_stream_reconnect/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G08 Run cases: processor_err_chan changefeed_reconstruct multi_capture synced_status_with_redo PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=d8b5b79e-24dd-4828-9a9d-4e1863df3394 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G08 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-6sdlp GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-6sdlp pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_err_chan/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Sun May 5 12:55:07 CST 2024] <<<<<< START cdc server in storage_csv_update case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.63556357.out server --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/storage_csv_update/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G09 Run cases: gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=c9787df3-2869-4af9-8230-35e4e653f28d BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G09 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-hpjnh GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-hpjnh pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/gc_safepoint/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + endpoints=https://127.0.0.1:2579 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates + [[ https://127.0.0.1:2579 =~ https ]] ++ cdc cli changefeed query --ca=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/ca.pem --cert=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client.pem --key=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_certificates/client-key.pem --pd=https://127.0.0.1:2579 -c custom-changefeed-name -s + info='{ "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546789087870983, "checkpoint_time": "2024-05-05 12:55:07.104", "error": null }' + echo '{ "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546789087870983, "checkpoint_time": "2024-05-05 12:55:07.104", "error": null }' { "upstream_id": 7365374355680606051, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546789087870983, "checkpoint_time": "2024-05-05 12:55:07.104", "error": null } ++ echo '{' '"upstream_id":' 7365374355680606051, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546789087870983, '"checkpoint_time":' '"2024-05-05' '12:55:07.104",' '"error":' null '}' ++ jq -r .state ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365374355680606051, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546789087870983, '"checkpoint_time":' '"2024-05-05' '12:55:07.104",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4224.out cli unsafe delete-service-gc-safepoint Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/sink_retry Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/kv_client_stream_reconnect Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... start tidb cluster in /tmp/tidb_cdc_test/row_format Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/charset_gbk Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:55:10 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffc0cb84-509d-4266-9dfa-8eba540d92a8 {"id":"ffc0cb84-509d-4266-9dfa-8eba540d92a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884907} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471adbb6c6 ffc0cb84-509d-4266-9dfa-8eba540d92a8 /tidb/cdc/default/default/upstream/7365374542210559975 {"id":7365374542210559975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffc0cb84-509d-4266-9dfa-8eba540d92a8 {"id":"ffc0cb84-509d-4266-9dfa-8eba540d92a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884907} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471adbb6c6 ffc0cb84-509d-4266-9dfa-8eba540d92a8 /tidb/cdc/default/default/upstream/7365374542210559975 {"id":7365374542210559975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ffc0cb84-509d-4266-9dfa-8eba540d92a8 {"id":"ffc0cb84-509d-4266-9dfa-8eba540d92a8","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884907} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471adbb6c6 ffc0cb84-509d-4266-9dfa-8eba540d92a8 /tidb/cdc/default/default/upstream/7365374542210559975 {"id":7365374542210559975,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_csv_update.cli.6418.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml Create changefeed successfully! ID: b86438a5-8430-4b11-b6f0-d315d02993bc Info: {"upstream_id":7365374542210559975,"namespace":"default","id":"b86438a5-8430-4b11-b6f0-d315d02993bc","sink_uri":"file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:55:10.514070873+08:00","start_ts":449546789945606149,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546789945606149,"checkpoint_ts":449546789945606149,"checkpoint_time":"2024-05-05 12:55:10.376"} PASS + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4256.out cli unsafe reset --no-confirm --pd=https://127.0.0.1:2579 coverage: 2.5% of statements in github.com/pingcap/tiflow/... reset and all metadata truncated in PD! PASS [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { start tidb cluster in /tmp/tidb_cdc_test/processor_err_chan Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] withEnv [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { [Pipeline] container [Pipeline] { Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) [Pipeline] stage [Pipeline] { (Test) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6c62c80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:3891, start at 2024-05-05 12:55:09.262069734 +0800 CST m=+5.289343690 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:09.269 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:09.234 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:09.234 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6c62c80012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:3891, start at 2024-05-05 12:55:09.262069734 +0800 CST m=+5.289343690 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:09.269 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:09.234 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:09.234 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6c6444000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:3980, start at 2024-05-05 12:55:09.345496148 +0800 CST m=+5.324830711 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:09.351 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:09.329 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:09.329 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { + set +x [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { [Pipeline] timeout Timeout set to expire in 40 min [Pipeline] { Verifying downstream PD is started... [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] withCredentials Masking supported pattern matches of $TICDC_COVERALLS_TOKEN or $TICDC_CODECOV_TOKEN [Pipeline] { [Pipeline] { Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] dir Running in /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + check_file_exists true + local all_should_exist=true + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' true == true ']' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/10.data ']' + SINK_URI='file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' + run_cdc_cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.storage_cleanup.cli.3946.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml Create changefeed successfully! ID: 88841d4b-4c64-4ddd-afcb-cf0f3d9fa544 Info: {"upstream_id":7365374442769582770,"namespace":"default","id":"88841d4b-4c64-4ddd-afcb-cf0f3d9fa544","sink_uri":"file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s","create_time":"2024-05-05T12:55:11.809510817+08:00","start_ts":449546790278791172,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"cloud_storage_config":{"file_expiration_days":1,"file_cleanup_cron_spec":"* * * * * *"},"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546790278791172,"checkpoint_ts":449546790278791172,"checkpoint_time":"2024-05-05 12:55:11.647"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5351.out cli tso query --pd=http://127.0.0.1:2379 Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/gc_safepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4372.out cli unsafe resolve-lock --region=62 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + set +x + tso='449546791048970241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546791048970241 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sun May 5 12:55:17 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.54015403.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_tls_with_auth.cli.4406.out cli unsafe resolve-lock --region=62 --ts=449546790490341381 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 1634 46 --:--:-- --:--:-- --:--:-- 1643 { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:55:20 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d152ecf1-19ed-4c94-9e63-03d4096ef519 {"id":"d152ecf1-19ed-4c94-9e63-03d4096ef519","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884917} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471aec58f5 d152ecf1-19ed-4c94-9e63-03d4096ef519 /tidb/cdc/default/default/upstream/7365374563294691735 {"id":7365374563294691735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d152ecf1-19ed-4c94-9e63-03d4096ef519 {"id":"d152ecf1-19ed-4c94-9e63-03d4096ef519","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884917} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471aec58f5 d152ecf1-19ed-4c94-9e63-03d4096ef519 /tidb/cdc/default/default/upstream/7365374563294691735 {"id":7365374563294691735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d152ecf1-19ed-4c94-9e63-03d4096ef519 {"id":"d152ecf1-19ed-4c94-9e63-03d4096ef519","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884917} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471aec58f5 d152ecf1-19ed-4c94-9e63-03d4096ef519 /tidb/cdc/default/default/upstream/7365374563294691735 {"id":7365374563294691735,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5459.out cli changefeed create --start-ts=449546791048970241 '--sink-uri=file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-598?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 53fee7e1-1ad9-4081-8cf6-dc84bf6ff7dc Info: {"upstream_id":7365374563294691735,"namespace":"default","id":"53fee7e1-1ad9-4081-8cf6-dc84bf6ff7dc","sink_uri":"file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-598?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:55:20.63236068+08:00","start_ts":449546791048970241,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546791048970241,"checkpoint_ts":449546791048970241,"checkpoint_time":"2024-05-05 12:55:14.585"} PASS ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d0af40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:1236, start at 2024-05-05 12:55:20.013379649 +0800 CST m=+5.333583978 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:20.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:19.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:19.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d03600005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp, pid:1229, start at 2024-05-05 12:55:19.516668595 +0800 CST m=+5.362707599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:19.524 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:19.512 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:19.512 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d03600005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp, pid:1229, start at 2024-05-05 12:55:19.516668595 +0800 CST m=+5.362707599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:19.524 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:19.512 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:19.512 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... + set +x + workdir=/tmp/tidb_cdc_test/ddl_puller_lag + sink_uri='file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-598?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_puller_lag ++ date + echo '[Sun May 5 12:55:22 CST 2024] <<<<<< START storage consumer in ddl_puller_lag case >>>>>>' [Sun May 5 12:55:22 CST 2024] <<<<<< START storage consumer in ddl_puller_lag case >>>>>> + cd /tmp/tidb_cdc_test/ddl_puller_lag + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_puller_lag + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_puller_lag/storage_test/ticdc-ddl-puller-lag-test-598?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d04d00002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp, pid:1310, start at 2024-05-05 12:55:19.606099336 +0800 CST m=+5.400837158 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:19.612 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:19.604 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:19.604 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kv_client_stream_reconnect/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 7.19 secs (518242110 bytes/sec) [Pipeline] { [Pipeline] cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d1ee40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:1240, start at 2024-05-05 12:55:21.299515613 +0800 CST m=+5.101657759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:21.307 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:21.273 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:21.273 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d1ee40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:1240, start at 2024-05-05 12:55:21.299515613 +0800 CST m=+5.101657759 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:21.307 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:21.273 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:21.273 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d212c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:1329, start at 2024-05-05 12:55:21.449696068 +0800 CST m=+5.185439567 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:21.458 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:21.419 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:21.419 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/row_format/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/row_format/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/row_format/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/row_format/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 181 100 181 0 0 1391 0 --:--:-- --:--:-- --:--:-- 1392 { "version": "v8.2.0-alpha-53-g0de8dc3e4", "git_hash": "0de8dc3e43ec741eba58047155ce7f3dba8eb4f7", "id": "b7253c7e-ff86-44a4-92a1-8cba89fa9da7", "pid": 3472, "is_owner": true ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d0af40010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:1236, start at 2024-05-05 12:55:20.013379649 +0800 CST m=+5.333583978 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:20.023 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:19.997 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:19.997 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d0ae80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:1320, start at 2024-05-05 12:55:20.023997657 +0800 CST m=+5.258136455 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:20.032 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:19.994 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:19.994 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_retry/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_retry/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_retry/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } }wait process cdc.test exit for 1-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d13c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5, pid:1231, start at 2024-05-05 12:55:20.604804942 +0800 CST m=+5.417777846 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:20.615 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:20.610 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:20.610 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d13c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5, pid:1231, start at 2024-05-05 12:55:20.604804942 +0800 CST m=+5.417777846 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:20.615 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:20.610 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:20.610 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d14740015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5, pid:1302, start at 2024-05-05 12:55:20.653815484 +0800 CST m=+5.404996554 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:20.659 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:20.654 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:20.654 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/charset_gbk/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/charset_gbk/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:55:24 CST 2024] <<<<<< run test case cli_tls_with_auth success! >>>>>> + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2652.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d27800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:1179, start at 2024-05-05 12:55:21.855683121 +0800 CST m=+5.329953611 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:21.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:21.824 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:21.824 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d27800012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:1179, start at 2024-05-05 12:55:21.855683121 +0800 CST m=+5.329953611 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:21.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:21.824 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:21.824 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d28fc0010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:1254, start at 2024-05-05 12:55:21.935397617 +0800 CST m=+5.348625198 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:21.942 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:21.919 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:21.919 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_err_chan/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_err_chan/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sun May 5 12:55:24 CST 2024] <<<<<< START cdc server in kv_client_stream_reconnect case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/kv/kvClientForceReconnect=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kv_client_stream_reconnect.26152617.out server --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2625.out cli tso query --pd=http://127.0.0.1:2379 [Sun May 5 12:55:25 CST 2024] <<<<<< START cdc server in charset_gbk case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.charset_gbk.26532655.out server --log-file /tmp/tidb_cdc_test/charset_gbk/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/charset_gbk/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + tso='449546793745907713 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546793745907713 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 12:55:26 CST 2024] <<<<<< START cdc server in row_format case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.26892691.out server --log-file /tmp/tidb_cdc_test/row_format/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/row_format/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449546793870426113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546793870426113 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "threadcount"="2" "operationcount"="0" "updateproportion"="0" "scanproportion"="0" "mysql.port"="4000" "readproportion"="0" "recordcount"="10" "insertproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "mysql.db"="sink_retry" "readallfields"="true" "dotransactions"="false" "mysql.user"="root" "requestdistribution"="uniform" ********************************************** Run finished, takes 9.09261ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1953.5, Avg(us): 1750, Min(us): 1106, Max(us): 3946, 95th(us): 4000, 99th(us): 4000 [Sun May 5 12:55:27 CST 2024] <<<<<< START cdc server in sink_retry case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkTxnRandomError=25%return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.26762678.out server --log-file /tmp/tidb_cdc_test/sink_retry/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_retry/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:55:27 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5235f585-960a-417c-9508-868db342ca23 {"id":"5235f585-960a-417c-9508-868db342ca23","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884924} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1be8cd 5235f585-960a-417c-9508-868db342ca23 /tidb/cdc/default/default/upstream/7365374610777717193 {"id":7365374610777717193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5235f585-960a-417c-9508-868db342ca23 {"id":"5235f585-960a-417c-9508-868db342ca23","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884924} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1be8cd 5235f585-960a-417c-9508-868db342ca23 /tidb/cdc/default/default/upstream/7365374610777717193 {"id":7365374610777717193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5235f585-960a-417c-9508-868db342ca23 {"id":"5235f585-960a-417c-9508-868db342ca23","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884924} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1be8cd 5235f585-960a-417c-9508-868db342ca23 /tidb/cdc/default/default/upstream/7365374610777717193 {"id":7365374610777717193,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/kv_client_stream_reconnect + sink_uri='file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/storage_test/kv-client-stream-reconnect-4678?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/kv_client_stream_reconnect ++ date + echo '[Sun May 5 12:55:27 CST 2024] <<<<<< START storage consumer in kv_client_stream_reconnect case >>>>>>' [Sun May 5 12:55:27 CST 2024] <<<<<< START storage consumer in kv_client_stream_reconnect case >>>>>> + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/kv_client_stream_reconnect + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/kv_client_stream_reconnect/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/kv_client_stream_reconnect/storage_test/kv-client-stream-reconnect-4678?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d736c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:1297, start at 2024-05-05 12:55:26.690846137 +0800 CST m=+6.061636872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:26.697 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:26.683 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:26.683 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d736c0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:1297, start at 2024-05-05 12:55:26.690846137 +0800 CST m=+6.061636872 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:26.697 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:26.683 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:26.683 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c6d67a40014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:1380, start at 2024-05-05 12:55:25.952517753 +0800 CST m=+5.246234282 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:57:25.959 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:55:25.929 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:45:25.929 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/gc_safepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/gc_safepoint/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:55:29 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a1b1ab0-a64d-4e6d-aeaa-95b72d394f94 {"id":"9a1b1ab0-a64d-4e6d-aeaa-95b72d394f94","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1bf3d3 9a1b1ab0-a64d-4e6d-aeaa-95b72d394f94 /tidb/cdc/default/default/upstream/7365374610034626105 {"id":7365374610034626105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a1b1ab0-a64d-4e6d-aeaa-95b72d394f94 {"id":"9a1b1ab0-a64d-4e6d-aeaa-95b72d394f94","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1bf3d3 9a1b1ab0-a64d-4e6d-aeaa-95b72d394f94 /tidb/cdc/default/default/upstream/7365374610034626105 {"id":7365374610034626105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9a1b1ab0-a64d-4e6d-aeaa-95b72d394f94 {"id":"9a1b1ab0-a64d-4e6d-aeaa-95b72d394f94","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884926} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1bf3d3 9a1b1ab0-a64d-4e6d-aeaa-95b72d394f94 /tidb/cdc/default/default/upstream/7365374610034626105 {"id":7365374610034626105,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.row_format.cli.2750.out cli changefeed create --start-ts=449546793745907713 '--sink-uri=file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-6387?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: faf52578-c154-4b07-956c-1df80132b07f Info: {"upstream_id":7365374610034626105,"namespace":"default","id":"faf52578-c154-4b07-956c-1df80132b07f","sink_uri":"file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-6387?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:55:29.818483095+08:00","start_ts":449546793745907713,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546793745907713,"checkpoint_ts":449546793745907713,"checkpoint_time":"2024-05-05 12:55:24.873"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:55:29 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e63ef20-b9e4-48c0-866d-04b6ae1bbfef {"id":"7e63ef20-b9e4-48c0-866d-04b6ae1bbfef","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b19f3cc 7e63ef20-b9e4-48c0-866d-04b6ae1bbfef /tidb/cdc/default/default/upstream/7365374614593969676 {"id":7365374614593969676,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e63ef20-b9e4-48c0-866d-04b6ae1bbfef {"id":"7e63ef20-b9e4-48c0-866d-04b6ae1bbfef","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b19f3cc 7e63ef20-b9e4-48c0-866d-04b6ae1bbfef /tidb/cdc/default/default/upstream/7365374614593969676 {"id":7365374614593969676,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7e63ef20-b9e4-48c0-866d-04b6ae1bbfef {"id":"7e63ef20-b9e4-48c0-866d-04b6ae1bbfef","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884925} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b19f3cc 7e63ef20-b9e4-48c0-866d-04b6ae1bbfef /tidb/cdc/default/default/upstream/7365374614593969676 {"id":7365374614593969676,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 89c6d0de-eb81-40e1-b606-297e1b1f904b Info: {"upstream_id":7365374614593969676,"namespace":"default","id":"89c6d0de-eb81-40e1-b606-297e1b1f904b","sink_uri":"mysql://normal:xxxxx@127.0.0.1:3306/","create_time":"2024-05-05T12:55:30.169780514+08:00","start_ts":449546793926262785,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546793926262785,"checkpoint_ts":449546793926262785,"checkpoint_time":"2024-05-05 12:55:25.561"} + workdir=/tmp/tidb_cdc_test/charset_gbk + sink_uri=mysql://normal:123456@127.0.0.1:3306/ + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/charset_gbk ++ date + echo '[Sun May 5 12:55:30 CST 2024] <<<<<< START storage consumer in charset_gbk case >>>>>>' [Sun May 5 12:55:30 CST 2024] <<<<<< START storage consumer in charset_gbk case >>>>>> + cd /tmp/tidb_cdc_test/charset_gbk + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/charset_gbk + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/charset_gbk/cdc_storage_consumer.log --log-level debug --upstream-uri mysql://normal:123456@127.0.0.1:3306/ --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:55:30 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4e2de2d-9c3e-46e5-973f-ddc1695bc45a {"id":"b4e2de2d-9c3e-46e5-973f-ddc1695bc45a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884927} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1a18ef b4e2de2d-9c3e-46e5-973f-ddc1695bc45a /tidb/cdc/default/default/upstream/7365374609278385403 {"id":7365374609278385403,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4e2de2d-9c3e-46e5-973f-ddc1695bc45a {"id":"b4e2de2d-9c3e-46e5-973f-ddc1695bc45a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884927} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1a18ef b4e2de2d-9c3e-46e5-973f-ddc1695bc45a /tidb/cdc/default/default/upstream/7365374609278385403 {"id":7365374609278385403,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/b4e2de2d-9c3e-46e5-973f-ddc1695bc45a {"id":"b4e2de2d-9c3e-46e5-973f-ddc1695bc45a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884927} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1a18ef b4e2de2d-9c3e-46e5-973f-ddc1695bc45a /tidb/cdc/default/default/upstream/7365374609278385403 {"id":7365374609278385403,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_retry.cli.2732.out cli changefeed create --start-ts=449546793870426113 '--sink-uri=file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-1388?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 2a0d7248-8091-45f6-92f3-4299dcd0b778 Info: {"upstream_id":7365374609278385403,"namespace":"default","id":"2a0d7248-8091-45f6-92f3-4299dcd0b778","sink_uri":"file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-1388?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:55:30.568655537+08:00","start_ts":449546793870426113,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546793870426113,"checkpoint_ts":449546793870426113,"checkpoint_time":"2024-05-05 12:55:25.348"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Sun May 5 12:55:30 CST 2024] <<<<<< START cdc server in gc_safepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectGcSafepointUpdateInterval=return(500)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.gc_safepoint.26382640.out server --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/gc_safepoint/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + workdir=/tmp/tidb_cdc_test/row_format + sink_uri='file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-6387?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/row_format ++ date + echo '[Sun May 5 12:55:31 CST 2024] <<<<<< START storage consumer in row_format case >>>>>>' [Sun May 5 12:55:31 CST 2024] <<<<<< START storage consumer in row_format case >>>>>> + cd /tmp/tidb_cdc_test/row_format + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/row_format + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/row_format/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/row_format/storage_test/ticdc-row-format-test-6387?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + set +x + workdir=/tmp/tidb_cdc_test/sink_retry + sink_uri='file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-1388?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/sink_retry ++ date + echo '[Sun May 5 12:55:32 CST 2024] <<<<<< START storage consumer in sink_retry case >>>>>>' [Sun May 5 12:55:32 CST 2024] <<<<<< START storage consumer in sink_retry case >>>>>> + cd /tmp/tidb_cdc_test/sink_retry + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/sink_retry + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/sink_retry/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/sink_retry/storage_test/ticdc-sink-retry-test-1388?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:55:33 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d8b5e87-3e34-406a-8a34-c7a3c0834b3a {"id":"7d8b5e87-3e34-406a-8a34-c7a3c0834b3a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884930} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b2d61cd 7d8b5e87-3e34-406a-8a34-c7a3c0834b3a /tidb/cdc/default/default/upstream/7365374635869284909 {"id":7365374635869284909,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d8b5e87-3e34-406a-8a34-c7a3c0834b3a {"id":"7d8b5e87-3e34-406a-8a34-c7a3c0834b3a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884930} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b2d61cd 7d8b5e87-3e34-406a-8a34-c7a3c0834b3a /tidb/cdc/default/default/upstream/7365374635869284909 {"id":7365374635869284909,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d8b5e87-3e34-406a-8a34-c7a3c0834b3a {"id":"7d8b5e87-3e34-406a-8a34-c7a3c0834b3a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884930} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b2d61cd 7d8b5e87-3e34-406a-8a34-c7a3c0834b3a /tidb/cdc/default/default/upstream/7365374635869284909 {"id":7365374635869284909,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/gc_safepoint + sink_uri='file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-15254?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/gc_safepoint ++ date + echo '[Sun May 5 12:55:33 CST 2024] <<<<<< START storage consumer in gc_safepoint case >>>>>>' [Sun May 5 12:55:33 CST 2024] <<<<<< START storage consumer in gc_safepoint case >>>>>> + cd /tmp/tidb_cdc_test/gc_safepoint + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/gc_safepoint + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/gc_safepoint/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-15254?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' 0 check diff failed 1-th time, retry later + check_file_exists false + local all_should_exist=false + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type/2024-05-04/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns_dummy/2024-05-04/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/binary_columns/2024-05-04/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_data_type/2022-01-01/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/multi_charset/2022-01-02/10.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/1.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/2.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/3.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/4.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/5.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/6.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/7.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/8.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/9.data ']' + for f in '${EXIST_FILES[@]}' + '[' '!' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test_default/test/binary_columns/2022-01-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/binary_columns/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_data_type_dummy/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test/multi_charset_dummy/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_data_type/2024-05-03/10.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/1.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/2.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/3.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/4.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/5.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/6.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/7.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/8.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/9.data ']' + for f in '${CLEANED_FILES[@]}' + '[' false == true ']' + '[' -f /tmp/tidb_cdc_test/storage_cleanup/storage_test/test2/multi_charset/2024-05-03/10.data ']' + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/data/schema.sql 127.0.0.1 4000 + run_sql_file /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/data/data.sql 127.0.0.1 4000 + run_storage_consumer /tmp/tidb_cdc_test/storage_cleanup 'file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml '' + workdir=/tmp/tidb_cdc_test/storage_cleanup + sink_uri='file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Sun May 5 12:55:34 CST 2024] <<<<<< START storage consumer in storage_cleanup case >>>>>>' [Sun May 5 12:55:34 CST 2024] <<<<<< START storage consumer in storage_cleanup case >>>>>> + cd /tmp/tidb_cdc_test/storage_cleanup + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/storage_cleanup + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/storage_cleanup/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/storage_cleanup/storage_test?flush-interval=5s' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/changefeed.toml + sleep 8 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_basic/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:55:35 CST 2024] <<<<<< run test case kafka_simple_basic success! >>>>>> check diff failed 2-th time, retry later [Sun May 5 12:55:35 CST 2024] <<<<<< START cdc server in processor_err_chan case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/ProcessorAddTableError=1*return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_err_chan.27632765.out server --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_err_chan/cdc_data --cluster-id default --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_err_chan/conf/server.toml --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:55:38 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/148a15b6-daa4-40ab-953c-6b9300f1dfbc {"id":"148a15b6-daa4-40ab-953c-6b9300f1dfbc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1d548e 148a15b6-daa4-40ab-953c-6b9300f1dfbc /tidb/cdc/default/default/upstream/7365374618912424007 {"id":7365374618912424007,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/148a15b6-daa4-40ab-953c-6b9300f1dfbc {"id":"148a15b6-daa4-40ab-953c-6b9300f1dfbc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1d548e 148a15b6-daa4-40ab-953c-6b9300f1dfbc /tidb/cdc/default/default/upstream/7365374618912424007 {"id":7365374618912424007,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/148a15b6-daa4-40ab-953c-6b9300f1dfbc {"id":"148a15b6-daa4-40ab-953c-6b9300f1dfbc","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884936} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471b1d548e 148a15b6-daa4-40ab-953c-6b9300f1dfbc /tidb/cdc/default/default/upstream/7365374618912424007 {"id":7365374618912424007,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/processor_err_chan + sink_uri='file:///tmp/tidb_cdc_test/processor_err_chan/storage_test/ticdc-processor-err-chan-20883?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_err_chan ++ date + echo '[Sun May 5 12:55:39 CST 2024] <<<<<< START storage consumer in processor_err_chan case >>>>>>' [Sun May 5 12:55:39 CST 2024] <<<<<< START storage consumer in processor_err_chan case >>>>>> + cd /tmp/tidb_cdc_test/processor_err_chan + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_err_chan + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_err_chan/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_err_chan/storage_test/ticdc-processor-err-chan-20883?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_state http://127.0.0.1:2379 fbb60d33-5c96-491f-ae55-1569f7ae420c normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=fbb60d33-5c96-491f-ae55-1569f7ae420c + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c fbb60d33-5c96-491f-ae55-1569f7ae420c -s + info='{ "upstream_id": 7365374618912424007, "namespace": "default", "id": "fbb60d33-5c96-491f-ae55-1569f7ae420c", "state": "normal", "checkpoint_tso": 449546797429293060, "checkpoint_time": "2024-05-05 12:55:38.924", "error": null }' + echo '{ "upstream_id": 7365374618912424007, "namespace": "default", "id": "fbb60d33-5c96-491f-ae55-1569f7ae420c", "state": "normal", "checkpoint_tso": 449546797429293060, "checkpoint_time": "2024-05-05 12:55:38.924", "error": null }' { "upstream_id": 7365374618912424007, "namespace": "default", "id": "fbb60d33-5c96-491f-ae55-1569f7ae420c", "state": "normal", "checkpoint_tso": 449546797429293060, "checkpoint_time": "2024-05-05 12:55:38.924", "error": null } ++ echo '{' '"upstream_id":' 7365374618912424007, '"namespace":' '"default",' '"id":' '"fbb60d33-5c96-491f-ae55-1569f7ae420c",' '"state":' '"normal",' '"checkpoint_tso":' 449546797429293060, '"checkpoint_time":' '"2024-05-05' '12:55:38.924",' '"error":' null '}' ++ jq -r .state check diff failed 3-th time, retry later + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365374618912424007, '"namespace":' '"default",' '"id":' '"fbb60d33-5c96-491f-ae55-1569f7ae420c",' '"state":' '"normal",' '"checkpoint_tso":' 449546797429293060, '"checkpoint_time":' '"2024-05-05' '12:55:38.924",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check diff failed 1-th time, retry later table row_format.finish_mark not exists for 1-th check, retry later check diff failed 4-th time, retry later check diff failed 2-th time, retry later table row_format.finish_mark not exists for 2-th check, retry later + check_sync_diff /tmp/tidb_cdc_test/storage_cleanup /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_cleanup/conf/diff_config.toml 100 check diff failed 1-th time, retry later Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 19.22 secs (193732567 bytes/sec) [Pipeline] { [Pipeline] cache check diff failed 5-th time, retry later check diff failed 3-th time, retry later ls /tmp/tidb_cdc_test/storage_csv_update/storage_test/test/test_update 449546790365298722 run task successfully start check update + workdir=/tmp/tidb_cdc_test/storage_csv_update + sink_uri='file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/storage_csv_update ++ date + echo '[Sun May 5 12:55:42 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>>' [Sun May 5 12:55:42 CST 2024] <<<<<< START storage consumer in storage_csv_update case >>>>>> + cd /tmp/tidb_cdc_test/storage_csv_update + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/storage_csv_update + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/storage_csv_update/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/storage_csv_update/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/storage_csv_update/conf/changefeed.toml table charset_gbk_test0.t0 exists table charset_gbk_test0.t1 exists table charset_gbk_test1.t0 exists table test.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later table row_format.finish_mark not exists for 3-th check, retry later table test.finish_mark not exists for 2-th check, retry later check diff successfully check_safepoint_forward http://127.0.0.1:2379 7365374635869284909 449546798795063301 449546796029444099 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_basic_avro/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:55:45 CST 2024] <<<<<< run test case kafka_simple_basic_avro success! >>>>>> run task successfully check diff failed 4-th time, retry later check_changefeed_state http://127.0.0.1:2379 73924530-5a7d-4971-8667-b50341e2ce61 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=73924530-5a7d-4971-8667-b50341e2ce61 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 73924530-5a7d-4971-8667-b50341e2ce61 -s table row_format.finish_mark not exists for 4-th check, retry later + info='{ "upstream_id": 7365374635869284909, "namespace": "default", "id": "73924530-5a7d-4971-8667-b50341e2ce61", "state": "stopped", "checkpoint_tso": 449546799319089159, "checkpoint_time": "2024-05-05 12:55:46.133", "error": null }' + echo '{ "upstream_id": 7365374635869284909, "namespace": "default", "id": "73924530-5a7d-4971-8667-b50341e2ce61", "state": "stopped", "checkpoint_tso": 449546799319089159, "checkpoint_time": "2024-05-05 12:55:46.133", "error": null }' { "upstream_id": 7365374635869284909, "namespace": "default", "id": "73924530-5a7d-4971-8667-b50341e2ce61", "state": "stopped", "checkpoint_tso": 449546799319089159, "checkpoint_time": "2024-05-05 12:55:46.133", "error": null } ++ echo '{' '"upstream_id":' 7365374635869284909, '"namespace":' '"default",' '"id":' '"73924530-5a7d-4971-8667-b50341e2ce61",' '"state":' '"stopped",' '"checkpoint_tso":' 449546799319089159, '"checkpoint_time":' '"2024-05-05' '12:55:46.133",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365374635869284909, '"namespace":' '"default",' '"id":' '"73924530-5a7d-4971-8667-b50341e2ce61",' '"state":' '"stopped",' '"checkpoint_tso":' 449546799319089159, '"checkpoint_time":' '"2024-05-05' '12:55:46.133",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365374635869284909 table test.finish_mark not exists for 3-th check, retry later check diff successfully + check_logs /tmp/tidb_cdc_test/storage_cleanup ++ date + echo '[Sun May 5 12:55:47 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>' [Sun May 5 12:55:47 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>> + stop_tidb_cluster check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:55:48 CST 2024] <<<<<< run test case kafka_simple_handle_key_only success! >>>>>> table row_format.finish_mark not exists for 5-th check, retry later table test.finish_mark exists check table exists success check diff successfully wait process cdc.test exit for 1-th time... run task successfully check diff failed 6-th time, retry later check_changefeed_state http://127.0.0.1:2379 73924530-5a7d-4971-8667-b50341e2ce61 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=73924530-5a7d-4971-8667-b50341e2ce61 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 73924530-5a7d-4971-8667-b50341e2ce61 -s wait process cdc.test exit for 2-th time... table row_format.finish_mark not exists for 6-th check, retry later check diff failed 1-th time, retry later + info='{ "upstream_id": 7365374635869284909, "namespace": "default", "id": "73924530-5a7d-4971-8667-b50341e2ce61", "state": "normal", "checkpoint_tso": 449546799319089159, "checkpoint_time": "2024-05-05 12:55:46.133", "error": null }' + echo '{ "upstream_id": 7365374635869284909, "namespace": "default", "id": "73924530-5a7d-4971-8667-b50341e2ce61", "state": "normal", "checkpoint_tso": 449546799319089159, "checkpoint_time": "2024-05-05 12:55:46.133", "error": null }' { "upstream_id": 7365374635869284909, "namespace": "default", "id": "73924530-5a7d-4971-8667-b50341e2ce61", "state": "normal", "checkpoint_tso": 449546799319089159, "checkpoint_time": "2024-05-05 12:55:46.133", "error": null } ++ echo '{' '"upstream_id":' 7365374635869284909, '"namespace":' '"default",' '"id":' '"73924530-5a7d-4971-8667-b50341e2ce61",' '"state":' '"normal",' '"checkpoint_tso":' 449546799319089159, '"checkpoint_time":' '"2024-05-05' '12:55:46.133",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365374635869284909, '"namespace":' '"default",' '"id":' '"73924530-5a7d-4971-8667-b50341e2ce61",' '"state":' '"normal",' '"checkpoint_tso":' 449546799319089159, '"checkpoint_time":' '"2024-05-05' '12:55:46.133",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_forward http://127.0.0.1:2379 7365374635869284909 449546799319089158 449546799319089159 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:55:51 CST 2024] <<<<<< run test case charset_gbk success! >>>>>> run task successfully =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_handle_key_only_avro/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:55:51 CST 2024] <<<<<< run test case kafka_simple_handle_key_only_avro success! >>>>>> check_changefeed_state http://127.0.0.1:2379 73924530-5a7d-4971-8667-b50341e2ce61 stopped null + endpoints=http://127.0.0.1:2379 + changefeed_id=73924530-5a7d-4971-8667-b50341e2ce61 + expected_state=stopped + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 73924530-5a7d-4971-8667-b50341e2ce61 -s + info='{ "upstream_id": 7365374635869284909, "namespace": "default", "id": "73924530-5a7d-4971-8667-b50341e2ce61", "state": "stopped", "checkpoint_tso": 449546799319089159, "checkpoint_time": "2024-05-05 12:55:46.133", "error": null }' + echo '{ "upstream_id": 7365374635869284909, "namespace": "default", "id": "73924530-5a7d-4971-8667-b50341e2ce61", "state": "stopped", "checkpoint_tso": 449546799319089159, "checkpoint_time": "2024-05-05 12:55:46.133", "error": null }' { "upstream_id": 7365374635869284909, "namespace": "default", "id": "73924530-5a7d-4971-8667-b50341e2ce61", "state": "stopped", "checkpoint_tso": 449546799319089159, "checkpoint_time": "2024-05-05 12:55:46.133", "error": null } ++ echo '{' '"upstream_id":' 7365374635869284909, '"namespace":' '"default",' '"id":' '"73924530-5a7d-4971-8667-b50341e2ce61",' '"state":' '"stopped",' '"checkpoint_tso":' 449546799319089159, '"checkpoint_time":' '"2024-05-05' '12:55:46.133",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365374635869284909, '"namespace":' '"default",' '"id":' '"73924530-5a7d-4971-8667-b50341e2ce61",' '"state":' '"stopped",' '"checkpoint_tso":' 449546799319089159, '"checkpoint_time":' '"2024-05-05' '12:55:46.133",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_changefeed_state http://127.0.0.1:2379 808bd5cf-061b-45df-a7ab-0a4192b06460 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=808bd5cf-061b-45df-a7ab-0a4192b06460 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 808bd5cf-061b-45df-a7ab-0a4192b06460 -s + info='{ "upstream_id": 7365374635869284909, "namespace": "default", "id": "808bd5cf-061b-45df-a7ab-0a4192b06460", "state": "normal", "checkpoint_tso": 449546800944381954, "checkpoint_time": "2024-05-05 12:55:52.333", "error": null }' + echo '{ "upstream_id": 7365374635869284909, "namespace": "default", "id": "808bd5cf-061b-45df-a7ab-0a4192b06460", "state": "normal", "checkpoint_tso": 449546800944381954, "checkpoint_time": "2024-05-05 12:55:52.333", "error": null }' { "upstream_id": 7365374635869284909, "namespace": "default", "id": "808bd5cf-061b-45df-a7ab-0a4192b06460", "state": "normal", "checkpoint_tso": 449546800944381954, "checkpoint_time": "2024-05-05 12:55:52.333", "error": null } ++ echo '{' '"upstream_id":' 7365374635869284909, '"namespace":' '"default",' '"id":' '"808bd5cf-061b-45df-a7ab-0a4192b06460",' '"state":' '"normal",' '"checkpoint_tso":' 449546800944381954, '"checkpoint_time":' '"2024-05-05' '12:55:52.333",' '"error":' null '}' ++ jq -r .state table row_format.finish_mark not exists for 7-th check, retry later check diff failed 2-th time, retry later + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365374635869284909, '"namespace":' '"default",' '"id":' '"808bd5cf-061b-45df-a7ab-0a4192b06460",' '"state":' '"normal",' '"checkpoint_tso":' 449546800944381954, '"checkpoint_time":' '"2024-05-05' '12:55:52.333",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully check_safepoint_equal http://127.0.0.1:2379 7365374635869284909 check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:55:53 CST 2024] <<<<<< run test case processor_err_chan success! >>>>>> table row_format.finish_mark not exists for 8-th check, retry later check diff successfully [Sun May 5 12:55:54 CST 2024] <<<<<< run test case storage_csv_update success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 9.13 secs (408001942 bytes/sec) [Pipeline] { [Pipeline] cache run task successfully table row_format.finish_mark not exists for 9-th check, retry later Changefeed remove successfully. ID: 73924530-5a7d-4971-8667-b50341e2ce61 CheckpointTs: 449546799319089159 SinkURI: file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-15254?protocol=canal-json&enable-tidb-extension=true check_safepoint_forward http://127.0.0.1:2379 7365374635869284909 449546801953636354 449546799319089159 449546800944381954 run task successfully table row_format.finish_mark not exists for 10-th check, retry later Changefeed remove successfully. ID: 808bd5cf-061b-45df-a7ab-0a4192b06460 CheckpointTs: 449546802478186499 SinkURI: file:///tmp/tidb_cdc_test/gc_safepoint/storage_test/ticdc-gc-safepoint-15254?protocol=canal-json&enable-tidb-extension=true check_safepoint_cleared http://127.0.0.1:2379 7365374635869284909 run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:56:00 CST 2024] <<<<<< run test case gc_safepoint success! >>>>>> table row_format.finish_mark not exists for 11-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_manager/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_claim_check/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:00 CST 2024] <<<<<< run test case kafka_simple_claim_check success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table row_format.finish_mark not exists for 12-th check, retry later table sink_retry.finish_mark_1 exists check diff successfully ***************** properties ***************** "recordcount"="10" "updateproportion"="0" "scanproportion"="0" "dotransactions"="false" "mysql.host"="127.0.0.1" "mysql.db"="sink_retry" "requestdistribution"="uniform" "workload"="core" "readallfields"="true" "operationcount"="0" "readproportion"="0" "insertproportion"="0" "mysql.port"="4000" "mysql.user"="root" "threadcount"="2" ********************************************** Run finished, takes 4.353681ms INSERT - Takes(s): 0.0, Count: 10, OPS: 3249.9, Avg(us): 755, Min(us): 513, Max(us): 1860, 95th(us): 2000, 99th(us): 2000 start tidb cluster in /tmp/tidb_cdc_test/csv_storage_basic Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/ddl_manager Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/kafka_simple_claim_check_avro/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:04 CST 2024] <<<<<< run test case kafka_simple_claim_check_avro success! >>>>>> table row_format.finish_mark not exists for 13-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_reconstruct/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff failed 1-th time, retry later check diff failed 2-th time, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table row_format.finish_mark not exists for 14-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_adapter_compatibility/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:07 CST 2024] <<<<<< run test case canal_json_adapter_compatibility success! >>>>>> table row_format.finish_mark not exists for 15-th check, retry later check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 16-th check, retry later check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_basic/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:11 CST 2024] <<<<<< run test case canal_json_basic success! >>>>>> table row_format.finish_mark not exists for 17-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_pause_resume/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff failed 5-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c70415c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:5098, start at 2024-05-05 12:56:12.654633705 +0800 CST m=+5.081477042 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:12.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:12.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:12.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c70415c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:5098, start at 2024-05-05 12:56:12.654633705 +0800 CST m=+5.081477042 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:12.662 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:12.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:12.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7042080014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:5173, start at 2024-05-05 12:56:12.706147218 +0800 CST m=+5.062858980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:12.712 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:12.674 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:12.674 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_reconstruct Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Starting Upstream TiFlash... Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 TiFlash Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 UTC Build Time: 2024-04-30 02:34:21 Verifying upstream PD is started... Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/csv_storage_basic/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/csv_storage_basic/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/csv_storage_basic/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 6-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_content_compatible/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:15 CST 2024] <<<<<< run test case canal_json_content_compatible success! >>>>>> VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c70638c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5, pid:3880, start at 2024-05-05 12:56:14.856034848 +0800 CST m=+5.135298730 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:14.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:14.819 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:14.819 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c70638c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5, pid:3880, start at 2024-05-05 12:56:14.856034848 +0800 CST m=+5.135298730 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:14.862 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:14.819 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:14.819 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... table row_format.finish_mark not exists for 18-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7062b00018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-20f-p79f5, pid:3961, start at 2024-05-05 12:56:14.814014879 +0800 CST m=+5.045164538 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:14.819 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:14.814 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:14.814 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_manager/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_manager/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sun May 5 12:56:15 CST 2024] <<<<<< START cdc server in csv_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.64886490.out server --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_basic/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 17.50 secs (212769625 bytes/sec) [Pipeline] { [Pipeline] cache Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/changefeed_pause_resume Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table row_format.finish_mark not exists for 19-th check, retry later check diff failed 7-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Sun May 5 12:56:18 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.52125214.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_topics/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:18 CST 2024] <<<<<< run test case multi_topics success! >>>>>> Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:56:18 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c904641a-f970-4f1c-b70e-2f07d1fa46d5 {"id":"c904641a-f970-4f1c-b70e-2f07d1fa46d5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471bee89c4 c904641a-f970-4f1c-b70e-2f07d1fa46d5 /tidb/cdc/default/default/upstream/7365374838964104445 {"id":7365374838964104445,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c904641a-f970-4f1c-b70e-2f07d1fa46d5 {"id":"c904641a-f970-4f1c-b70e-2f07d1fa46d5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471bee89c4 c904641a-f970-4f1c-b70e-2f07d1fa46d5 /tidb/cdc/default/default/upstream/7365374838964104445 {"id":7365374838964104445,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/c904641a-f970-4f1c-b70e-2f07d1fa46d5 {"id":"c904641a-f970-4f1c-b70e-2f07d1fa46d5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884976} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471bee89c4 c904641a-f970-4f1c-b70e-2f07d1fa46d5 /tidb/cdc/default/default/upstream/7365374838964104445 {"id":7365374838964104445,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_basic.cli.6550.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml Create changefeed successfully! ID: 80c27909-2202-4730-87be-70ab7a51e7e6 Info: {"upstream_id":7365374838964104445,"namespace":"default","id":"80c27909-2202-4730-87be-70ab7a51e7e6","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s","create_time":"2024-05-05T12:56:19.394806334+08:00","start_ts":449546808008638469,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546808008638469,"checkpoint_ts":449546808008638469,"checkpoint_time":"2024-05-05 12:56:19.281"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... table row_format.finish_mark not exists for 20-th check, retry later wait process cdc.test exit for 1-th time... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL table row_format.finish_mark not exists for 21-th check, retry later * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:56:21 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61361fa3-8f64-458b-b510-3c47d273ebb0 {"id":"61361fa3-8f64-458b-b510-3c47d273ebb0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471beee0cb 61361fa3-8f64-458b-b510-3c47d273ebb0 /tidb/cdc/default/default/upstream/7365374840870513468 {"id":7365374840870513468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61361fa3-8f64-458b-b510-3c47d273ebb0 {"id":"61361fa3-8f64-458b-b510-3c47d273ebb0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471beee0cb 61361fa3-8f64-458b-b510-3c47d273ebb0 /tidb/cdc/default/default/upstream/7365374840870513468 {"id":7365374840870513468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/61361fa3-8f64-458b-b510-3c47d273ebb0 {"id":"61361fa3-8f64-458b-b510-3c47d273ebb0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884978} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471beee0cb 61361fa3-8f64-458b-b510-3c47d273ebb0 /tidb/cdc/default/default/upstream/7365374840870513468 {"id":7365374840870513468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.cli.5273.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json&enable-tidb-extension=true' -c=ddl-manager + set +x + workdir=/tmp/tidb_cdc_test/csv_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_basic ++ date + echo '[Sun May 5 12:56:21 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>>' [Sun May 5 12:56:21 CST 2024] <<<<<< START storage consumer in csv_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_basic/storage_test?flush-interval=5s' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_basic/conf/changefeed.toml cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:56:21 CST 2024] <<<<<< run test case kv_client_stream_reconnect success! >>>>>> Create changefeed successfully! ID: ddl-manager Info: {"upstream_id":7365374840870513468,"namespace":"default","id":"ddl-manager","sink_uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:56:21.573982866+08:00","start_ts":449546808581947398,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546808581947398,"checkpoint_ts":449546808581947398,"checkpoint_time":"2024-05-05 12:56:21.468"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + workdir=/tmp/tidb_cdc_test/ddl_manager + sink_uri='file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_manager ++ date + echo '[Sun May 5 12:56:23 CST 2024] <<<<<< START storage consumer in ddl_manager case >>>>>>' [Sun May 5 12:56:23 CST 2024] <<<<<< START storage consumer in ddl_manager case >>>>>> + cd /tmp/tidb_cdc_test/ddl_manager + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_manager + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_manager/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table row_format.finish_mark not exists for 22-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c70f1540017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:4171, start at 2024-05-05 12:56:23.939611025 +0800 CST m=+5.153923051 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:23.945 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:23.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:23.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c70f1540017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:4171, start at 2024-05-05 12:56:23.939611025 +0800 CST m=+5.153923051 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:23.945 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:23.943 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:23.943 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c70f3a0000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:4259, start at 2024-05-05 12:56:24.053172446 +0800 CST m=+5.197249514 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:24.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:24.040 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:24.040 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_reconstruct/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table row_format.finish_mark not exists for 23-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c71020c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:4593, start at 2024-05-05 12:56:24.975945957 +0800 CST m=+5.173084393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:24.982 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:24.963 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:24.963 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c71020c000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:4593, start at 2024-05-05 12:56:24.975945957 +0800 CST m=+5.173084393 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:24.982 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:24.963 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:24.963 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7102d00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:4677, start at 2024-05-05 12:56:25.054214371 +0800 CST m=+5.199832860 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:25.061 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:25.061 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:25.061 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_pause_resume/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/avro_basic/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:27 CST 2024] <<<<<< run test case avro_basic success! >>>>>> [Sun May 5 12:56:27 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.56315633.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver1 --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 24-th check, retry later [Sun May 5 12:56:28 CST 2024] <<<<<< START cdc server in changefeed_pause_resume case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_pause_resume.60516053.out server --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table row_format.finish_mark not exists for 25-th check, retry later check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:56:30 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfd343e4-2279-4c23-b4f3-507e47576a8b {"id":"cfd343e4-2279-4c23-b4f3-507e47576a8b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884987} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c1285c3 cfd343e4-2279-4c23-b4f3-507e47576a8b /tidb/cdc/default/default/upstream/7365374885255824712 {"id":7365374885255824712,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfd343e4-2279-4c23-b4f3-507e47576a8b {"id":"cfd343e4-2279-4c23-b4f3-507e47576a8b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884987} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c1285c3 cfd343e4-2279-4c23-b4f3-507e47576a8b /tidb/cdc/default/default/upstream/7365374885255824712 {"id":7365374885255824712,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfd343e4-2279-4c23-b4f3-507e47576a8b {"id":"cfd343e4-2279-4c23-b4f3-507e47576a8b","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884987} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c1285c3 cfd343e4-2279-4c23-b4f3-507e47576a8b /tidb/cdc/default/default/upstream/7365374885255824712 {"id":7365374885255824712,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/changefeed_reconstruct + sink_uri='file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-9252?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_reconstruct ++ date + echo '[Sun May 5 12:56:30 CST 2024] <<<<<< START storage consumer in changefeed_reconstruct case >>>>>>' [Sun May 5 12:56:30 CST 2024] <<<<<< START storage consumer in changefeed_reconstruct case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_reconstruct + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_reconstruct + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-9252?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' ***************** properties ***************** "readproportion"="0" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "insertproportion"="0" "requestdistribution"="uniform" "threadcount"="4" "dotransactions"="false" "readallfields"="true" "workload"="core" "updateproportion"="0" "mysql.user"="root" "recordcount"="50" "scanproportion"="0" "mysql.db"="changefeed_reconstruct" ********************************************** Run finished, takes 19.889086ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3073.6, Avg(us): 1546, Min(us): 983, Max(us): 4213, 95th(us): 5000, 99th(us): 5000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_handle_key_only/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:30 CST 2024] <<<<<< run test case canal_json_handle_key_only success! >>>>>> + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:56:31 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3752cd89-c82e-47c4-b1b6-d155cf137e8d {"id":"3752cd89-c82e-47c4-b1b6-d155cf137e8d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884988} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c1e51c3 3752cd89-c82e-47c4-b1b6-d155cf137e8d /tidb/cdc/default/default/upstream/7365374890692489554 {"id":7365374890692489554,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3752cd89-c82e-47c4-b1b6-d155cf137e8d {"id":"3752cd89-c82e-47c4-b1b6-d155cf137e8d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884988} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c1e51c3 3752cd89-c82e-47c4-b1b6-d155cf137e8d /tidb/cdc/default/default/upstream/7365374890692489554 {"id":7365374890692489554,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3752cd89-c82e-47c4-b1b6-d155cf137e8d {"id":"3752cd89-c82e-47c4-b1b6-d155cf137e8d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714884988} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c1e51c3 3752cd89-c82e-47c4-b1b6-d155cf137e8d /tidb/cdc/default/default/upstream/7365374890692489554 {"id":7365374890692489554,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/changefeed_pause_resume + sink_uri='file:///tmp/tidb_cdc_test/changefeed_pause_resume/storage_test/ticdc-changefeed-pause-resume-29832?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_pause_resume ++ date + echo '[Sun May 5 12:56:31 CST 2024] <<<<<< START storage consumer in changefeed_pause_resume case >>>>>>' [Sun May 5 12:56:31 CST 2024] <<<<<< START storage consumer in changefeed_pause_resume case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_pause_resume + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_pause_resume + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_pause_resume/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_pause_resume/storage_test/ticdc-changefeed-pause-resume-29832?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table row_format.finish_mark not exists for 26-th check, retry later table changefeed_pause_resume.t1 not exists for 1-th check, retry later check diff failed 2-th time, retry later table changefeed_reconstruct.usertable not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cdc/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:32 CST 2024] <<<<<< run test case cdc success! >>>>>> table row_format.finish_mark exists check diff successfully table changefeed_pause_resume.t1 not exists for 2-th check, retry later check diff successfully [Sun May 5 12:56:33 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>> wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/open_protocol_handle_key_only/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:33 CST 2024] <<<<<< run test case open_protocol_handle_key_only success! >>>>>> wait process cdc.test exit for 2-th time... table changefeed_reconstruct.usertable not exists for 3-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:56:34 CST 2024] <<<<<< run test case row_format success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 16.89 secs (220498376 bytes/sec) [Pipeline] { [Pipeline] cache table sink_retry.finish_mark_2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:56:34 CST 2024] <<<<<< run test case sink_retry success! >>>>>> =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/split_region/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table changefeed_pause_resume.t1 not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_claim_check/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:36 CST 2024] <<<<<< run test case canal_json_claim_check success! >>>>>> table changefeed_reconstruct.usertable not exists for 4-th check, retry later table changefeed_reconstruct.usertable not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/split_region Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table changefeed_pause_resume.t1 not exists for 4-th check, retry later table changefeed_reconstruct.usertable exists check diff successfully wait process 5217 exit for 1-th time... wait process 5217 exit for 2-th time... wait process 5217 exit for 3-th time... wait process 5636 exit for 1-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process 5217 exit for 4-th time... table changefeed_pause_resume.t1 not exists for 5-th check, retry later wait process 5636 exit for 2-th time... wait process 5217 exit for 5-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5636) - No such process wait process 5636 exit for 3-th time... process 5636 already exit check_no_capture http://127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully [Sun May 5 12:56:42 CST 2024] <<<<<< START cdc server in changefeed_reconstruct case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_reconstruct.59225924.out server --log-file /tmp/tidb_cdc_test/changefeed_reconstruct/cdcserver2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_reconstruct/cdc_dataserver2 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (5217) - No such process wait process 5217 exit for 6-th time... process 5217 already exit [Sun May 5 12:56:42 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.53985400.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_pause_resume.t1 exists table changefeed_pause_resume.t2 exists table changefeed_pause_resume.t3 exists =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/open_protocol_claim_check/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:56:43 CST 2024] <<<<<< run test case open_protocol_claim_check success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 6.11 secs (609276961 bytes/sec) [Pipeline] { [Pipeline] cache ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 1-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:56:45 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/c0f43f58-9566-4f17-9775-b6713d3f78fa {UpstreamID:7365374885255824712 Namespace:default ID:c0f43f58-9566-4f17-9775-b6713d3f78fa SinkURI:file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-9252?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:56:30.376278196 +0800 CST StartTs:449546810895368194 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001a946c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546810908475398} {CheckpointTs:449546813399105545 MinTableBarrierTs:449546814709825544 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449546813399105545, checkpointTs: 449546813399105545, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09e4af61-77c1-4069-9439-5492f789d89d {"id":"09e4af61-77c1-4069-9439-5492f789d89d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c12868b 09e4af61-77c1-4069-9439-5492f789d89d /tidb/cdc/default/default/changefeed/info/c0f43f58-9566-4f17-9775-b6713d3f78fa {"upstream-id":7365374885255824712,"namespace":"default","changefeed-id":"c0f43f58-9566-4f17-9775-b6713d3f78fa","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-9252?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:56:30.376278196+08:00","start-ts":449546810895368194,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546810908475398} /tidb/cdc/default/default/changefeed/status/c0f43f58-9566-4f17-9775-b6713d3f78fa {"checkpoint-ts":449546813399105545,"min-table-barrier-ts":449546814709825544,"admin-job-type":0} /tidb/cdc/default/default/task/position/09e4af61-77c1-4069-9439-5492f789d89d/c0f43f58-9566-4f17-9775-b6713d3f78fa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365374885255824712 {"id":7365374885255824712,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c0f43f58-9566-4f17-9775-b6713d3f78fa {UpstreamID:7365374885255824712 Namespace:default ID:c0f43f58-9566-4f17-9775-b6713d3f78fa SinkURI:file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-9252?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:56:30.376278196 +0800 CST StartTs:449546810895368194 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001a946c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546810908475398} {CheckpointTs:449546813399105545 MinTableBarrierTs:449546814709825544 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449546813399105545, checkpointTs: 449546813399105545, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09e4af61-77c1-4069-9439-5492f789d89d {"id":"09e4af61-77c1-4069-9439-5492f789d89d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c12868b 09e4af61-77c1-4069-9439-5492f789d89d /tidb/cdc/default/default/changefeed/info/c0f43f58-9566-4f17-9775-b6713d3f78fa {"upstream-id":7365374885255824712,"namespace":"default","changefeed-id":"c0f43f58-9566-4f17-9775-b6713d3f78fa","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-9252?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:56:30.376278196+08:00","start-ts":449546810895368194,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546810908475398} /tidb/cdc/default/default/changefeed/status/c0f43f58-9566-4f17-9775-b6713d3f78fa {"checkpoint-ts":449546813399105545,"min-table-barrier-ts":449546814709825544,"admin-job-type":0} /tidb/cdc/default/default/task/position/09e4af61-77c1-4069-9439-5492f789d89d/c0f43f58-9566-4f17-9775-b6713d3f78fa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,+ grep -q 'failed to get info:' "warning":null} /tidb/cdc/default/default/upstream/7365374885255824712 {"id":7365374885255824712,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/c0f43f58-9566-4f17-9775-b6713d3f78fa {UpstreamID:7365374885255824712 Namespace:default ID:c0f43f58-9566-4f17-9775-b6713d3f78fa SinkURI:file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-9252?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:56:30.376278196 +0800 CST StartTs:449546810895368194 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc001a946c0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546810908475398} {CheckpointTs:449546813399105545 MinTableBarrierTs:449546814709825544 AdminJobType:noop} span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449546813399105545, checkpointTs: 449546813399105545, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/09e4af61-77c1-4069-9439-5492f789d89d {"id":"09e4af61-77c1-4069-9439-5492f789d89d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c12868b 09e4af61-77c1-4069-9439-5492f789d89d /tidb/cdc/default/default/changefeed/info/c0f43f58-9566-4f17-9775-b6713d3f78fa {"upstream-id":7365374885255824712,"namespace":"default","changefeed-id":"c0f43f58-9566-4f17-9775-b6713d3f78fa","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_reconstruct/storage_test/ticdc-changefeed-reconstruct-9252?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:56:30.376278196+08:00","start-ts":449546810895368194,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546810908475398} /tidb/cdc/default/default/changefeed/status/c0f43f58-9566-4f17-9775-b6713d3f78fa {"checkpoint-ts":449546813399105545,"min-table-barrier-ts":449546814709825544,"admin-job-type":0} /tidb/cdc/default/default/task/position/09e4af61-77c1-4069-9439-5492f789d89d/c0f43f58-9566-4f17-9775-b6713d3f78fa {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365374885255824712 {"id":7365374885255824712,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc.test cli capture list --pd=http://127.0.0.1:2379 2>&1 | grep id + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_error/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:56:46 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365374840870513468 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:56:21.573982866 +0800 CST StartTs:449546808581947398 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00247afc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546808608423938} {CheckpointTs:449546809669844998 MinTableBarrierTs:449546809669844998 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15511b6a-72fd-47c3-8993-50d8659f4066 {"id":"15511b6a-72fd-47c3-8993-50d8659f4066","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471beee3b9 15511b6a-72fd-47c3-8993-50d8659f4066 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365374840870513468,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:56:21.573982866+08:00","start-ts":449546808581947398,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546808608423938} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449546809669844998,"min-table-barrier-ts":449546809669844998,"admin-job-type":0} /tidb/cdc/default/default/task/position/15511b6a-72fd-47c3-8993-50d8659f4066/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365374840870513468 {"id":7365374840870513468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365374840870513468 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:56:21.573982866 +0800 CST StartTs:449546808581947398 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00247afc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546808608423938} {CheckpointTs:449546809669844998 MinTableBarrierTs:449546809669844998 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15511b6a-72fd-47c3-8993-50d8659f4066 {"id":"15511b6a-72fd-47c3-8993-50d8659f4066","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471beee3b9 15511b6a-72fd-47c3-8993-50d8659f4066 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365374840870513468,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:56:21.573982866+08:00","start-ts":449546808581947398,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546808608423938} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449546809669844998,"min-table-barrier-ts":449546809669844998,"admin-job-type":0} /tidb/cdc/default/default/task/position/15511b6a-72fd-47c3-8993-50d8659f4066/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365374840870513468 {"id":7365374840870513468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365374840870513468 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:56:21.573982866 +0800 CST StartTs:449546808581947398 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00247afc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546808608423938} {CheckpointTs:449546809669844998 MinTableBarrierTs:449546809669844998 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15511b6a-72fd-47c3-8993-50d8659f4066 {"id":"15511b6a-72fd-47c3-8993-50d8659f4066","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471beee3b9 15511b6a-72fd-47c3-8993-50d8659f4066 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365374840870513468,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:56:21.573982866+08:00","start-ts":449546808581947398,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546808608423938} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449546809669844998,"min-table-barrier-ts":449546809669844998,"admin-job-type":0} /tidb/cdc/default/default/task/position/15511b6a-72fd-47c3-8993-50d8659f4066/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365374840870513468 {"id":7365374840870513468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun May 5 12:56:46 CST 2024] <<<<<< START cdc server in ddl_manager case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteDDLSlowly=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_manager.54575459.out server --log-file /tmp/tidb_cdc_test/ddl_manager/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_manager/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... "id": "09e4af61-77c1-4069-9439-5492f789d89d", "cluster-id": "default" run task successfully < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:56:47 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365374840870513468 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:56:21.573982866 +0800 CST StartTs:449546808581947398 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00247afc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546808608423938} {CheckpointTs:449546809669844998 MinTableBarrierTs:449546809669844998 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15511b6a-72fd-47c3-8993-50d8659f4066 {"id":"15511b6a-72fd-47c3-8993-50d8659f4066","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471beee3b9 15511b6a-72fd-47c3-8993-50d8659f4066 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365374840870513468,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:56:21.573982866+08:00","start-ts":449546808581947398,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546808608423938} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449546809669844998,"min-table-barrier-ts":449546809800917017,"admin-job-type":0} /tidb/cdc/default/default/task/position/15511b6a-72fd-47c3-8993-50d8659f4066/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365374840870513468 {"id":7365374840870513468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365374840870513468 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:56:21.573982866 +0800 CST StartTs:449546808581947398 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00247afc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546808608423938} {CheckpointTs:449546809669844998 MinTableBarrierTs:449546809669844998 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15511b6a-72fd-47c3-8993-50d8659f4066 {"id":"15511b6a-72fd-47c3-8993-50d8659f4066","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471beee3b9 15511b6a-72fd-47c3-8993-50d8659f4066 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365374840870513468,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:56:21.573982866+08:00","start-ts":449546808581947398,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546808608423938} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449546809669844998,"min-table-barrier-ts":449546809800917017,"admin-job-type":0} /tidb/cdc/default/default/task/position/15511b6a-72fd-47c3-8993-50d8659f4066/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365374840870513468 {"id":7365374840870513468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-manager {UpstreamID:7365374840870513468 Namespace:default ID:ddl-manager SinkURI:file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:56:21.573982866 +0800 CST StartTs:449546808581947398 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00247afc0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546808608423938} {CheckpointTs:449546809669844998 MinTableBarrierTs:449546809669844998 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/15511b6a-72fd-47c3-8993-50d8659f4066 {"id":"15511b6a-72fd-47c3-8993-50d8659f4066","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885002} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471beee3b9 15511b6a-72fd-47c3-8993-50d8659f4066 /tidb/cdc/default/default/changefeed/info/ddl-manager {"upstream-id":7365374840870513468,"namespace":"default","changefeed-id":"ddl-manager","sink-uri":"file:///tmp/tidb_cdc_test/ddl_manager/storage_test/ticdc-ddl-mamager-test-17353?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:56:21.573982866+08:00","start-ts":449546808581947398,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546808608423938} /tidb/cdc/default/default/changefeed/status/ddl-manager {"checkpoint-ts":449546809669844998,"min-table-barrier-ts":449546809800917017,"admin-job-type":0} /tidb/cdc/default/default/task/position/15511b6a-72fd-47c3-8993-50d8659f4066/ddl-manager {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365374840870513468 {"id":7365374840870513468,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table ddl_manager.finish_mark not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 2-th time, retry later capture_id: 09e4af61-77c1-4069-9439-5492f789d89d check_processor_table_count http://127.0.0.1:2379 c0f43f58-9566-4f17-9775-b6713d3f78fa 09e4af61-77c1-4069-9439-5492f789d89d 1 run task successfully check_processor_table_count http://127.0.0.1:2379 c0f43f58-9566-4f17-9775-b6713d3f78fa 09e4af61-77c1-4069-9439-5492f789d89d 0 start tidb cluster in /tmp/tidb_cdc_test/changefeed_error Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table count 1 does equal to expected count 0 run task failed 1-th time, retry later check diff failed 3-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c72722c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp, pid:4461, start at 2024-05-05 12:56:48.547412714 +0800 CST m=+5.434390618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:48.555 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:48.523 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:48.523 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/drop_many_tables/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check_processor_table_count http://127.0.0.1:2379 c0f43f58-9566-4f17-9775-b6713d3f78fa 09e4af61-77c1-4069-9439-5492f789d89d 0 check diff failed 4-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c72722c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp, pid:4461, start at 2024-05-05 12:56:48.547412714 +0800 CST m=+5.434390618 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:48.555 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:48.523 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:48.523 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c72720c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7f6-f38hp, pid:4542, start at 2024-05-05 12:56:48.546455447 +0800 CST m=+5.366825502 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:48.556 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:48.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:48.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/split_region/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/split_region/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/split_region/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/split_region/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } run task successfully table ddl_manager.finish_mark not exists for 3-th check, retry later ***************** properties ***************** "recordcount"="50" "insertproportion"="0" "mysql.host"="127.0.0.1" "scanproportion"="0" "requestdistribution"="uniform" "updateproportion"="0" "mysql.db"="changefeed_reconstruct" "dotransactions"="false" "readallfields"="true" "workload"="core" "threadcount"="4" "mysql.port"="4000" "operationcount"="0" "readproportion"="0" "mysql.user"="root" ********************************************** Run finished, takes 16.246766ms INSERT - Takes(s): 0.0, Count: 48, OPS: 3745.9, Avg(us): 1277, Min(us): 851, Max(us): 3621, 95th(us): 4000, 99th(us): 4000 table changefeed_reconstruct.usertable not exists for 1-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release start tidb cluster in /tmp/tidb_cdc_test/drop_many_tables Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.5789.out cli tso query --pd=http://127.0.0.1:2379 table changefeed_reconstruct.usertable not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_basic Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x + tso='449546817050509313 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546817050509313 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 5-th check, retry later [Sun May 5 12:56:55 CST 2024] <<<<<< START cdc server in split_region case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.58335835.out server --log-file /tmp/tidb_cdc_test/split_region/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/split_region/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later table changefeed_reconstruct.usertable not exists for 3-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 6-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 7-th time, retry later table changefeed_reconstruct.usertable not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:56:58 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53d02965-1ce9-4319-b73d-5f2c7aede8d0 {"id":"53d02965-1ce9-4319-b73d-5f2c7aede8d0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885015} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c776af8 53d02965-1ce9-4319-b73d-5f2c7aede8d0 /tidb/cdc/default/default/upstream/7365374991428490724 {"id":7365374991428490724,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53d02965-1ce9-4319-b73d-5f2c7aede8d0 {"id":"53d02965-1ce9-4319-b73d-5f2c7aede8d0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885015} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c776af8 53d02965-1ce9-4319-b73d-5f2c7aede8d0 /tidb/cdc/default/default/upstream/7365374991428490724 {"id":7365374991428490724,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53d02965-1ce9-4319-b73d-5f2c7aede8d0 {"id":"53d02965-1ce9-4319-b73d-5f2c7aede8d0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885015} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c776af8 53d02965-1ce9-4319-b73d-5f2c7aede8d0 /tidb/cdc/default/default/upstream/7365374991428490724 {"id":7365374991428490724,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.split_region.cli.5888.out cli changefeed create --start-ts=449546817050509313 '--sink-uri=file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-16045?protocol=canal-json&enable-tidb-extension=true' --config /tmp/tidb_cdc_test/split_region/pulsar_test.toml Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: 548000f8-740c-418c-ab83-7131f52a0e83 Info: {"upstream_id":7365374991428490724,"namespace":"default","id":"548000f8-740c-418c-ab83-7131f52a0e83","sink_uri":"file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-16045?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:56:59.07437524+08:00","start_ts":449546817050509313,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546817050509313,"checkpoint_ts":449546817050509313,"checkpoint_time":"2024-05-05 12:56:53.773"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 8-th time, retry later table ddl_manager.finish_mark not exists for 7-th check, retry later table changefeed_reconstruct.usertable not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7312fc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:3905, start at 2024-05-05 12:56:58.817685506 +0800 CST m=+5.320047016 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:58.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:58.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:58.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... + set +x + workdir=/tmp/tidb_cdc_test/split_region + sink_uri='file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-16045?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/split_region ++ date + echo '[Sun May 5 12:57:00 CST 2024] <<<<<< START storage consumer in split_region case >>>>>>' [Sun May 5 12:57:00 CST 2024] <<<<<< START storage consumer in split_region case >>>>>> + cd /tmp/tidb_cdc_test/split_region + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/split_region + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/split_region/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/split_region/storage_test/ticdc-split-region-test-16045?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table split_region.test1 not exists for 1-th check, retry later Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 8-th check, retry later table changefeed_reconstruct.usertable exists check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7312fc0002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:3905, start at 2024-05-05 12:56:58.817685506 +0800 CST m=+5.320047016 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:58.828 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:58.815 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:58.815 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7313b40013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:3986, start at 2024-05-05 12:56:58.888869488 +0800 CST m=+5.323933754 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:58:58.896 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:56:58.861 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:46:58.861 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_error/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_error/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 9-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 1-th time... table split_region.test1 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:57:03 CST 2024] <<<<<< run test case changefeed_reconstruct success! >>>>>> table ddl_manager.finish_mark not exists for 9-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5310.out cli tso query --pd=http://127.0.0.1:2379 check diff successfully VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c735c100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:3926, start at 2024-05-05 12:57:03.536944998 +0800 CST m=+5.269662713 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:03.547 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:03.542 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:03.542 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table split_region.test1 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 10-th check, retry later + set +x + tso='449546819774709761 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546819774709761 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x ***************** properties ***************** "updateproportion"="0" "readproportion"="0" "mysql.port"="4000" "insertproportion"="0" "mysql.db"="changefeed_error" "threadcount"="4" "scanproportion"="0" "operationcount"="0" "workload"="core" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "recordcount"="20" "dotransactions"="false" "mysql.user"="root" "readallfields"="true" ********************************************** Run finished, takes 9.630098ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3828.1, Avg(us): 1834, Min(us): 1046, Max(us): 4356, 95th(us): 5000, 99th(us): 5000 [Sun May 5 12:57:05 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedNoRetryError=1*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.53745376.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c735c100014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:3926, start at 2024-05-05 12:57:03.536944998 +0800 CST m=+5.269662713 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:03.547 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:03.542 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:03.542 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c735ce00010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:4009, start at 2024-05-05 12:57:03.560712558 +0800 CST m=+5.224695056 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:03.571 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:03.544 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:03.544 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c737c140009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:6603, start at 2024-05-05 12:57:05.550725363 +0800 CST m=+5.230424849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:05.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:05.541 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:05.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table split_region.test1 not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7380000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:7617, start at 2024-05-05 12:57:05.822937868 +0800 CST m=+5.139940015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:05.833 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:05.792 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:05.792 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7380000014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:7617, start at 2024-05-05 12:57:05.822937868 +0800 CST m=+5.139940015 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:05.833 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:05.792 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:05.792 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c738314000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:7695, start at 2024-05-05 12:57:06.000887572 +0800 CST m=+5.255687599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:06.011 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:05.989 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:05.989 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 11-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c737c140009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:6603, start at 2024-05-05 12:57:05.550725363 +0800 CST m=+5.230424849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:05.557 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:05.541 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:05.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c737cd00017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:6686, start at 2024-05-05 12:57:05.622527181 +0800 CST m=+5.249061245 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:05.630 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:05.588 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:05.588 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_storage_basic/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_storage_basic/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table split_region.test1 not exists for 5-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5396.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:08 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f {"id":"fec8f38e-ec5e-4f08-9f0a-026b41dc417f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885026} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e0eee fec8f38e-ec5e-4f08-9f0a-026b41dc417f /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f {"id":"fec8f38e-ec5e-4f08-9f0a-026b41dc417f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885026} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e0eee fec8f38e-ec5e-4f08-9f0a-026b41dc417f /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f {"id":"fec8f38e-ec5e-4f08-9f0a-026b41dc417f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885026} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e0eee fec8f38e-ec5e-4f08-9f0a-026b41dc417f /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5440.out cli changefeed create --start-ts=449546819774709761 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error Create changefeed successfully! ID: changefeed-error Info: {"upstream_id":7365375037484554602,"namespace":"default","id":"changefeed-error","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:57:09.363792906+08:00","start_ts":449546819774709761,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546819774709761,"checkpoint_ts":449546819774709761,"checkpoint_time":"2024-05-05 12:57:04.165"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Sun May 5 12:57:09 CST 2024] <<<<<< START cdc server in csv_storage_multi_tables_ddl case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_multi_tables_ddl.91189120.out server --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 12-th check, retry later + set +x + tso='449546821026971649 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546821026971649 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 12:57:10 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.54345436.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table split_region.test1 not exists for 6-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/changefeed_error + sink_uri='file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_error ++ date + echo '[Sun May 5 12:57:10 CST 2024] <<<<<< START storage consumer in changefeed_error case >>>>>>' [Sun May 5 12:57:10 CST 2024] <<<<<< START storage consumer in changefeed_error case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_error + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_error + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_error/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_state http://127.0.0.1:2379 changefeed-error failed [CDC:ErrStartTsBeforeGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=failed + error_msg='[CDC:ErrStartTsBeforeGC]' + tls_dir='[CDC:ErrStartTsBeforeGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s + info='{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449546819774709761, "checkpoint_time": "2024-05-05 12:57:04.165", "error": { "time": "2024-05-05T12:57:09.436580078+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449546819774709461 is earlier than or equal to GC safepoint at 449546819774709761" } }' + echo '{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449546819774709761, "checkpoint_time": "2024-05-05 12:57:04.165", "error": { "time": "2024-05-05T12:57:09.436580078+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449546819774709461 is earlier than or equal to GC safepoint at 449546819774709761" } }' { "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-error", "state": "failed", "checkpoint_tso": 449546819774709761, "checkpoint_time": "2024-05-05 12:57:04.165", "error": { "time": "2024-05-05T12:57:09.436580078+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrStartTsBeforeGC", "message": "[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449546819774709461 is earlier than or equal to GC safepoint at 449546819774709761" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449546819774709761, '"checkpoint_time":' '"2024-05-05' '12:57:04.165",' '"error":' '{' '"time":' '"2024-05-05T12:57:09.436580078+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449546819774709461 is earlier than or equal to GC safepoint at '449546819774709761"' '}' '}' [Sun May 5 12:57:10 CST 2024] <<<<<< START cdc server in canal_json_storage_basic case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.80658067.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"failed",' '"checkpoint_tso":' 449546819774709761, '"checkpoint_time":' '"2024-05-05' '12:57:04.165",' '"error":' '{' '"time":' '"2024-05-05T12:57:09.436580078+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrStartTsBeforeGC",' '"message":' '"[CDC:ErrStartTsBeforeGC]fail' to create or maintain changefeed because start-ts 449546819774709461 is earlier than or equal to GC safepoint at '449546819774709761"' '}' '}' + message='[CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449546819774709461 is earlier than or equal to GC safepoint at 449546819774709761' + [[ ! [CDC:ErrStartTsBeforeGC]fail to create or maintain changefeed because start-ts 449546819774709461 is earlier than or equal to GC safepoint at 449546819774709761 =~ \[CDC:ErrStartTsBeforeGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5561.out cli changefeed resume -c changefeed-error check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 13-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:12 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f67a6e69-5b44-4c0c-b804-969c9898eae4 {"id":"f67a6e69-5b44-4c0c-b804-969c9898eae4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471cb630c4 f67a6e69-5b44-4c0c-b804-969c9898eae4 /tidb/cdc/default/default/upstream/7365375066399605070 {"id":7365375066399605070,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f67a6e69-5b44-4c0c-b804-969c9898eae4 {"id":"f67a6e69-5b44-4c0c-b804-969c9898eae4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471cb630c4 f67a6e69-5b44-4c0c-b804-969c9898eae4 /tidb/cdc/default/default/upstream/7365375066399605070 {"id":7365375066399605070,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f67a6e69-5b44-4c0c-b804-969c9898eae4 {"id":"f67a6e69-5b44-4c0c-b804-969c9898eae4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885029} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471cb630c4 f67a6e69-5b44-4c0c-b804-969c9898eae4 /tidb/cdc/default/default/upstream/7365375066399605070 {"id":7365375066399605070,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: test-normal Info: {"upstream_id":7365375066399605070,"namespace":"default","id":"test-normal","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-24108?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-05T12:57:12.346714197+08:00","start_ts":449546821053186049,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546821053186049,"checkpoint_ts":449546821053186049,"checkpoint_time":"2024-05-05 12:57:09.042"} Create changefeed successfully! ID: test-error-1 Info: {"upstream_id":7365375066399605070,"namespace":"default","id":"test-error-1","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-18323?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-05T12:57:12.495649448+08:00","start_ts":449546821053186049,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546821053186049,"checkpoint_ts":449546821053186049,"checkpoint_time":"2024-05-05 12:57:09.042"} Create changefeed successfully! ID: test-error-2 Info: {"upstream_id":7365375066399605070,"namespace":"default","id":"test-error-2","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-14612?flush-interval=5s\u0026protocol=csv","create_time":"2024-05-05T12:57:12.645772487+08:00","start_ts":449546821053186049,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546821053186049,"checkpoint_ts":449546821053186049,"checkpoint_time":"2024-05-05 12:57:09.042"} + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-24108?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + log_suffix=1 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Sun May 5 12:57:12 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Sun May 5 12:57:12 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer1.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-normal-24108?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/normal.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-18323?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + log_suffix=2 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Sun May 5 12:57:12 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Sun May 5 12:57:12 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer2.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-1-18323?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-1.toml + workdir=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-14612?flush-interval=5s&protocol=csv' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml + log_suffix=3 ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_multi_tables_ddl ++ date + echo '[Sun May 5 12:57:12 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>>' [Sun May 5 12:57:12 CST 2024] <<<<<< START storage consumer in csv_storage_multi_tables_ddl case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/cdc_storage_consumer3.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_multi_tables_ddl/storage_test/ticdc-multi-tables-ddl-test-error-2-14612?flush-interval=5s&protocol=csv' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/conf/error-2.toml table split_region.test1 exists table split_region.test2 exists check diff successfully + set +x table changefeed_error.usertable not exists for 1-th check, retry later check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 14-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:13 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e05bb9e-81e3-4d2c-bb40-8076c73863e6 {"id":"5e05bb9e-81e3-4d2c-bb40-8076c73863e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885030} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471caaa9d8 5e05bb9e-81e3-4d2c-bb40-8076c73863e6 /tidb/cdc/default/default/upstream/7365375050093038718 {"id":7365375050093038718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e05bb9e-81e3-4d2c-bb40-8076c73863e6 {"id":"5e05bb9e-81e3-4d2c-bb40-8076c73863e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885030} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471caaa9d8 5e05bb9e-81e3-4d2c-bb40-8076c73863e6 /tidb/cdc/default/default/upstream/7365375050093038718 {"id":7365375050093038718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5e05bb9e-81e3-4d2c-bb40-8076c73863e6 {"id":"5e05bb9e-81e3-4d2c-bb40-8076c73863e6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885030} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471caaa9d8 5e05bb9e-81e3-4d2c-bb40-8076c73863e6 /tidb/cdc/default/default/upstream/7365375050093038718 {"id":7365375050093038718,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5493.out cli changefeed create --start-ts=449546821026971649 '--sink-uri=file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-22927?protocol=canal-json&enable-tidb-extension=true' + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:13 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16ac224a-aabb-4462-aa39-f98e5776b8af {"id":"16ac224a-aabb-4462-aa39-f98e5776b8af","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885031} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471cb2acd1 16ac224a-aabb-4462-aa39-f98e5776b8af /tidb/cdc/default/default/upstream/7365375055464536617 {"id":7365375055464536617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16ac224a-aabb-4462-aa39-f98e5776b8af {"id":"16ac224a-aabb-4462-aa39-f98e5776b8af","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885031} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471cb2acd1 16ac224a-aabb-4462-aa39-f98e5776b8af /tidb/cdc/default/default/upstream/7365375055464536617 {"id":7365375055464536617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/16ac224a-aabb-4462-aa39-f98e5776b8af {"id":"16ac224a-aabb-4462-aa39-f98e5776b8af","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885031} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471cb2acd1 16ac224a-aabb-4462-aa39-f98e5776b8af /tidb/cdc/default/default/upstream/7365375055464536617 {"id":7365375055464536617,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_basic.cli.8125.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml Create changefeed successfully! ID: d9438570-4e53-4775-9144-6582e4fbaf1f Info: {"upstream_id":7365375050093038718,"namespace":"default","id":"d9438570-4e53-4775-9144-6582e4fbaf1f","sink_uri":"file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-22927?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:57:13.896836904+08:00","start_ts":449546821026971649,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546821026971649,"checkpoint_ts":449546821026971649,"checkpoint_time":"2024-05-05 12:57:08.942"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Create changefeed successfully! ID: 20a4f89b-d9e0-447c-960b-a923ad57e5c7 Info: {"upstream_id":7365375055464536617,"namespace":"default","id":"20a4f89b-d9e0-447c-960b-a923ad57e5c7","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:57:14.17462798+08:00","start_ts":449546822377013250,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546822377013250,"checkpoint_ts":449546822377013250,"checkpoint_time":"2024-05-05 12:57:14.092"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_manager.finish_mark not exists for 15-th check, retry later + set +x table changefeed_error.usertable not exists for 2-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/drop_many_tables + sink_uri='file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-22927?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/drop_many_tables ++ date + echo '[Sun May 5 12:57:15 CST 2024] <<<<<< START storage consumer in drop_many_tables case >>>>>>' [Sun May 5 12:57:15 CST 2024] <<<<<< START storage consumer in drop_many_tables case >>>>>> + cd /tmp/tidb_cdc_test/drop_many_tables + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/drop_many_tables + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/drop_many_tables/storage_test/ticdc-drop-tables-test-22927?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table multi_tables_ddl_test.t55 not exists for 1-th check, retry later + workdir=/tmp/tidb_cdc_test/canal_json_storage_basic + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_basic ++ date + echo '[Sun May 5 12:57:15 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>>' [Sun May 5 12:57:15 CST 2024] <<<<<< START storage consumer in canal_json_storage_basic case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_basic + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_basic + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_basic/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_basic/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_basic/conf/changefeed.toml check diff failed 5-th time, retry later table drop_tables.c not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 16-th check, retry later table changefeed_error.usertable not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 2-th check, retry later check diff failed 6-th time, retry later table drop_tables.c not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_capture/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 31.40 secs (118584866 bytes/sec) [Pipeline] { [Pipeline] cache table changefeed_error.usertable not exists for 4-th check, retry later check diff failed 7-th time, retry later table ddl_manager.finish_mark not exists for 17-th check, retry later table drop_tables.c not exists for 3-th check, retry later table multi_tables_ddl_test.t55 not exists for 3-th check, retry later table changefeed_error.usertable exists check diff successfully ***************** properties ***************** "requestdistribution"="uniform" "workload"="core" "scanproportion"="0" "updateproportion"="0" "operationcount"="0" "threadcount"="4" "mysql.port"="4000" "readproportion"="0" "mysql.host"="127.0.0.1" "dotransactions"="false" "mysql.db"="changefeed_error" "readallfields"="true" "insertproportion"="0" "recordcount"="20" "mysql.user"="root" ********************************************** Run finished, takes 12.780063ms INSERT - Takes(s): 0.0, Count: 20, OPS: 6981.7, Avg(us): 2455, Min(us): 445, Max(us): 9941, 95th(us): 10000, 99th(us): 10000 check diff failed 8-th time, retry later check diff successfully {"id":"fec8f38e-ec5e-4f08-9f0a-026b41dc417f","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885026} check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f + [[ /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f + echo 'check failed' check failed + exit 1 run task failed 1-th time, retry later table ddl_manager.finish_mark not exists for 18-th check, retry later table drop_tables.c not exists for 4-th check, retry later table multi_tables_ddl_test.t55 not exists for 4-th check, retry later check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f + [[ /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f + echo 'check failed' check failed + exit 1 run task failed 2-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/multi_capture Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 19-th check, retry later check diff failed 1-th time, retry later check diff successfully table drop_tables.c not exists for 5-th check, retry later table multi_tables_ddl_test.t55 not exists for 5-th check, retry later check diff failed 1-th time, retry later Verifying downstream PD is started... check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 20-th check, retry later check diff failed 2-th time, retry later table multi_tables_ddl_test.t55 not exists for 6-th check, retry later table drop_tables.c not exists for 6-th check, retry later check diff failed 2-th time, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info=/tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f + [[ /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f =~ capture ]] + echo 'capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f' capture contains in etcd /tidb/cdc/default/__cdc_meta__/capture/fec8f38e-ec5e-4f08-9f0a-026b41dc417f + echo 'check failed' check failed + exit 1 run task failed 3-th time, retry later check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 21-th check, retry later check diff successfully [Sun May 5 12:57:28 CST 2024] <<<<<< run test case canal_json_storage_basic success! >>>>>> table drop_tables.c not exists for 7-th check, retry later table multi_tables_ddl_test.t55 not exists for 7-th check, retry later check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 22-th check, retry later table drop_tables.c not exists for 8-th check, retry later table multi_tables_ddl_test.t55 not exists for 8-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... check diff failed 4-th time, retry later table ddl_manager.finish_mark not exists for 23-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit table drop_tables.c not exists for 9-th check, retry later [Sun May 5 12:57:32 CST 2024] <<<<<< run test case split_region success! >>>>>> table multi_tables_ddl_test.t55 not exists for 9-th check, retry later table ddl_manager.finish_mark not exists for 24-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/capture' 'capture' + key_prefix=/tidb/cdc/default/__cdc_meta__/capture + message=capture ++ etcdctl get /tidb/cdc/default/__cdc_meta__/capture --prefix --keys-only + info= + [[ '' =~ capture ]] + echo 'check pass' check pass + exit 0 run task successfully check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sun May 5 12:57:33 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/NewChangefeedRetryError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.57985800.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 5-th time, retry later table drop_tables.c not exists for 10-th check, retry later table multi_tables_ddl_test.t55 exists table multi_tables_ddl_test.t66 exists table multi_tables_ddl_test.t7 exists table multi_tables_ddl_test.t88 exists table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 25-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7543440010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:7237, start at 2024-05-05 12:57:34.689251909 +0800 CST m=+5.237946416 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:34.697 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:34.673 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:34.673 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:36 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a53a10a2-d2d9-4e3e-b4a5-8ff2573bcfc1 {"id":"a53a10a2-d2d9-4e3e-b4a5-8ff2573bcfc1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885053} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e0ffb a53a10a2-d2d9-4e3e-b4a5-8ff2573bcfc1 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365375037484554602,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:57:09.363792906+08:00","start-ts":449546819774709761,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-05T12:57:34.171494423+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546827639029761} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449546824073871365,"min-table-barrier-ts":449546824073871368,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a53a10a2-d2d9-4e3e-b4a5-8ff2573bcfc1 {"id":"a53a10a2-d2d9-4e3e-b4a5-8ff2573bcfc1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885053} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e0ffb a53a10a2-d2d9-4e3e-b4a5-8ff2573bcfc1 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365375037484554602,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:57:09.363792906+08:00","start-ts":449546819774709761,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-05T12:57:34.171494423+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546827639029761} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449546824073871365,"min-table-barrier-ts":449546824073871368,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a53a10a2-d2d9-4e3e-b4a5-8ff2573bcfc1 {"id":"a53a10a2-d2d9-4e3e-b4a5-8ff2573bcfc1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885053} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 3 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e0ffb a53a10a2-d2d9-4e3e-b4a5-8ff2573bcfc1 /tidb/cdc/default/default/changefeed/info/changefeed-error {"upstream-id":7365375037484554602,"namespace":"default","changefeed-id":"changefeed-error","sink-uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:57:09.363792906+08:00","start-ts":449546819774709761,"target-ts":0,"admin-job-type":1,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"pending","error":{"time":"2024-05-05T12:57:34.171494423+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrOwnerUnknown","message":"failpoint injected retriable error"},"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546827639029761} /tidb/cdc/default/default/changefeed/status/changefeed-error {"checkpoint-ts":449546824073871365,"min-table-barrier-ts":449546824073871368,"admin-job-type":1} /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error warning failpoint injected retriable error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error + expected_state=warning + error_msg=failpoint + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error -s table drop_tables.c exists check diff successfully + info='{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449546824073871365, "checkpoint_time": "2024-05-05 12:57:20.565", "error": { "time": "2024-05-05T12:57:34.171494423+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' + echo '{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449546824073871365, "checkpoint_time": "2024-05-05 12:57:20.565", "error": { "time": "2024-05-05T12:57:34.171494423+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } }' { "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-error", "state": "warning", "checkpoint_tso": 449546824073871365, "checkpoint_time": "2024-05-05 12:57:20.565", "error": { "time": "2024-05-05T12:57:34.171494423+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrOwnerUnknown", "message": "failpoint injected retriable error" } } ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449546824073871365, '"checkpoint_time":' '"2024-05-05' '12:57:20.565",' '"error":' '{' '"time":' '"2024-05-05T12:57:34.171494423+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-error",' '"state":' '"warning",' '"checkpoint_tso":' 449546824073871365, '"checkpoint_time":' '"2024-05-05' '12:57:20.565",' '"error":' '{' '"time":' '"2024-05-05T12:57:34.171494423+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrOwnerUnknown",' '"message":' '"failpoint' injected retriable 'error"' '}' '}' ++ jq -r .error.message + message='failpoint injected retriable error' + [[ ! failpoint injected retriable error =~ failpoint ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.5906.out cli changefeed remove -c changefeed-error wait process cdc.test exit for 1-th time... table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7543440010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:7237, start at 2024-05-05 12:57:34.689251909 +0800 CST m=+5.237946416 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:34.697 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:34.673 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:34.673 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7544bc0005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:7319, start at 2024-05-05 12:57:34.770947287 +0800 CST m=+5.241502684 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:34.778 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:34.767 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:34.767 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_capture/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_capture/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_capture/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 2-th time... Changefeed remove successfully. ID: changefeed-error CheckpointTs: 449546824073871365 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 7-th time, retry later table ddl_manager.finish_mark not exists for 26-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:57:38 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>> Cache restored successfully (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc) 3723625472 bytes in 17.01 secs (218898706 bytes/sec) [Pipeline] { [Pipeline] sh table multi_tables_ddl_test.finish_mark not exists for 3-th check, retry later [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G15 Run cases: new_ci_collation batch_add_table multi_rocks PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=282dd6a4-273d-4229-8ea8-0d8b4d95cd82 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G15 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-k2zh5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44 pingcap_tiflow_pull_cdc_integration_storage_test_1801-k2zh5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/new_ci_collation/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:57:39 CST 2024] <<<<<< run test case new_ci_collation success! >>>>>> [Pipeline] sh + set +x check_no_changefeed 127.0.0.1:2379 parse error: Invalid numeric literal at line 1, column 6 run task successfully + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G13 Run cases: tiflash region_merge common_1 PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=dfe70173-2418-4948-9149-b0c78cae51a7 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G13 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-7fvb6 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-7fvb6 pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/tiflash/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G11 Run cases: resolve_lock move_table autorandom generate_column PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=748048ca-6400-4b70-b3cb-9a2adc840148 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G11 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-g9d6f GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-g9d6f pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/resolve_lock/run.sh using Sink-Type: storage... <<================= [Pipeline] sh wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 27-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G14 Run cases: changefeed_finish force_replicate_table PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=9a3c918d-4734-4ab3-8944-61f03336a893 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G14 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-kkzwj GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw pingcap_tiflow_pull_cdc_integration_storage_test_1801-kkzwj GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_finish/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... [Pipeline] sh + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8650.out cli tso query --pd=http://127.0.0.1:2379 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G10 Run cases: default_value simple cdc_server_tips event_filter sql_mode PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=1194d87c-8e37-440c-8f69-2891dd5fe640 BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G10 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-59rmb GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-59rmb pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/default_value/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:57:40 CST 2024] <<<<<< run test case default_value success! >>>>>> wait process cdc.test exit for 2-th time... [Pipeline] sh check diff failed 8-th time, retry later [Pipeline] sh cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sun May 5 12:57:40 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/InjectChangefeedDDLError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.60006002.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G16 Run cases: owner_resign processor_etcd_worker_delay sink_hang PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=820aaf43-5412-4e4b-8ffb-19cef7b83daf BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G16 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-m7bnk GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-m7bnk pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/owner_resign/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table multi_tables_ddl_test.finish_mark not exists for 4-th check, retry later + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G12 Run cases: many_pk_or_uk capture_session_done_during_task ddl_attributes PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=1b3c5380-784e-42c2-9619-b6fd03803d2b BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51 HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT_443_TCP_PORT=443 KUBERNETES_PORT=tcp://10.233.0.1:443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G12 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-sx9p5 GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51 pingcap_tiflow_pull_cdc_integration_storage_test_1801-sx9p5 GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51 GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/many_pk_or_uk/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:57:41 CST 2024] <<<<<< run test case many_pk_or_uk success! >>>>>> + rm -rf /tmp/tidb_cdc_test + mkdir -p /tmp/tidb_cdc_test + chmod +x ./tests/integration_tests/run_group.sh + ./tests/integration_tests/run_group.sh storage G17 Run cases: clustered_index processor_resolved_ts_fallback PROW_JOB_ID=f567d06b-6ff2-4878-a444-f84e4f48b736 JENKINS_NODE_COOKIE=3a2df577-c0b1-432a-b96f-07a6a0a847df BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/ GOLANG_VERSION=1.21.6 HOSTNAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp HUDSON_SERVER_COOKIE=83ef27fe9acccc92 KUBERNETES_PORT=tcp://10.233.0.1:443 KUBERNETES_PORT_443_TCP_PORT=443 TERM=xterm STAGE_NAME=Test BUILD_TAG=jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801 KUBERNETES_SERVICE_PORT=443 GIT_PREVIOUS_COMMIT=03312178c534dce949face80c69812d989e55009 JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/pull_cdc_integration_storage_test","buildid":"1786980986911723522","prowjobid":"f567d06b-6ff2-4878-a444-f84e4f48b736","refs":{"org":"pingcap","repo":"tiflow","repo_link":"https://github.com/pingcap/tiflow","base_ref":"master","base_sha":"be1553484fe4c03594eabb8d7435c694e5fd7224","base_link":"https://github.com/pingcap/tiflow/commit/be1553484fe4c03594eabb8d7435c694e5fd7224","pulls":[{"number":10919,"author":"lidezhu","sha":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","title":"*(ticdc): split old update kv entry after restarting changefeed","link":"https://github.com/pingcap/tiflow/pull/10919","commit_link":"https://github.com/pingcap/tiflow/pull/10919/commits/0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","author_link":"https://github.com/lidezhu"}]}} KUBERNETES_SERVICE_HOST=10.233.0.1 WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/ RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=changes RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=artifacts FILE_SERVER_URL=http://fileserver.pingcap.net JENKINS_HOME=/var/jenkins_home GIT_COMMIT=03312178c534dce949face80c69812d989e55009 PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/../../scripts/bin RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct POD_CONTAINER=golang PWD=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow HUDSON_URL=https://do.pingcap.net/jenkins/ TICDC_COVERALLS_TOKEN=**** JOB_NAME=pingcap/tiflow/pull_cdc_integration_storage_test TZ=Asia/Shanghai BUILD_DISPLAY_NAME=#1801 TEST_GROUP=G17 JENKINS_URL=https://do.pingcap.net/jenkins/ BUILD_ID=1786980986911723522 TICDC_CODECOV_TOKEN=**** GOLANG_DOWNLOAD_SHA256=3f934f40ac360b9c01f616a9aa1796d227d8b0328bf64cb045c7b8c4ee9caea4 JOB_BASE_NAME=pull_cdc_integration_storage_test GIT_PREVIOUS_SUCCESSFUL_COMMIT=03312178c534dce949face80c69812d989e55009 RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/1801/display/redirect?page=tests SHLVL=5 HOME=/home/jenkins POD_LABEL=pingcap_tiflow_pull_cdc_integration_storage_test_1801-gn6zg GOROOT=/usr/local/go GIT_BRANCH=origin/main KUBERNETES_PORT_443_TCP_PROTO=tcp CI=true KUBERNETES_SERVICE_PORT_HTTPS=443 WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test@tmp EXECUTOR_NUMBER=0 JENKINS_SERVER_COOKIE=durable-4a523405106a7d00374130edbe2665c6e1f4e192f859497a5647eb99843c7c0e NODE_LABELS=pingcap_tiflow_pull_cdc_integration_storage_test_1801-gn6zg pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp GIT_URL=https://github.com/PingCAP-QE/ci.git HUDSON_HOME=/var/jenkins_home CLASSPATH= NODE_NAME=pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp GOPATH=/go JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/pull_cdc_integration_storage_test/display/redirect BUILD_NUMBER=1801 KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1 KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443 GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.6.linux-amd64.tar.gz _=/usr/bin/env find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/clustered_index/run.sh using Sink-Type: storage... <<================= [Sun May 5 12:57:41 CST 2024] <<<<<< skip test case clustered_index for storage! >>>>>> find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_resolved_ts_fallback/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + set +x + tso='449546829200883713 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546829200883713 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 28-th check, retry later ***************** properties ***************** "threadcount"="2" "requestdistribution"="uniform" "updateproportion"="0" "recordcount"="10" "mysql.db"="multi_capture_1" "workload"="core" "readproportion"="0" "scanproportion"="0" "mysql.port"="4000" "readallfields"="true" "mysql.user"="root" "dotransactions"="false" "insertproportion"="0" "mysql.host"="127.0.0.1" "operationcount"="0" ********************************************** Run finished, takes 9.126275ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2037.1, Avg(us): 1754, Min(us): 1017, Max(us): 4193, 95th(us): 5000, 99th(us): 5000 ***************** properties ***************** "readproportion"="0" "updateproportion"="0" "readallfields"="true" "insertproportion"="0" "recordcount"="10" "dotransactions"="false" "threadcount"="2" "scanproportion"="0" "operationcount"="0" "requestdistribution"="uniform" "workload"="core" "mysql.port"="4000" "mysql.db"="multi_capture_2" "mysql.host"="127.0.0.1" "mysql.user"="root" ********************************************** Run finished, takes 8.517216ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2037.6, Avg(us): 1619, Min(us): 866, Max(us): 3527, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "workload"="core" "mysql.db"="multi_capture_3" "readproportion"="0" "updateproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "insertproportion"="0" "recordcount"="10" "scanproportion"="0" "dotransactions"="false" "operationcount"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.user"="root" "readallfields"="true" ********************************************** Run finished, takes 8.907432ms INSERT - Takes(s): 0.0, Count: 10, OPS: 2002.9, Avg(us): 1686, Min(us): 1065, Max(us): 3791, 95th(us): 4000, 99th(us): 4000 ***************** properties ***************** "scanproportion"="0" "updateproportion"="0" "threadcount"="2" "operationcount"="0" "mysql.port"="4000" "readproportion"="0" "workload"="core" "requestdistribution"="uniform" "mysql.host"="127.0.0.1" "insertproportion"="0" "readallfields"="true" "mysql.db"="multi_capture_4" "dotransactions"="false" "mysql.user"="root" "recordcount"="10" ********************************************** Run finished, takes 9.044715ms INSERT - Takes(s): 0.0, Count: 10, OPS: 1966.4, Avg(us): 1735, Min(us): 1003, Max(us): 3938, 95th(us): 4000, 99th(us): 4000 [Sun May 5 12:57:42 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.87698771.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/batch_add_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/tiflash Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... The 1 times to try to start tidb cluster... table multi_tables_ddl_test.finish_mark not exists for 5-th check, retry later table ddl_manager.finish_mark not exists for 29-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:43 GMT < Content-Length: 885 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7397d7e3-c1c3-43fc-8f18-9afd59cc663e {"id":"7397d7e3-c1c3-43fc-8f18-9afd59cc663e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885061} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e1052 7397d7e3-c1c3-43fc-8f18-9afd59cc663e /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7397d7e3-c1c3-43fc-8f18-9afd59cc663e {"id":"7397d7e3-c1c3-43fc-8f18-9afd59cc663e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885061} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e1052 7397d7e3-c1c3-43fc-8f18-9afd59cc663e /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7397d7e3-c1c3-43fc-8f18-9afd59cc663e {"id":"7397d7e3-c1c3-43fc-8f18-9afd59cc663e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885061} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 4 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e1052 7397d7e3-c1c3-43fc-8f18-9afd59cc663e /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6055.out cli changefeed create --start-ts=449546819774709761 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-1 Create changefeed successfully! ID: changefeed-error-1 Info: {"upstream_id":7365375037484554602,"namespace":"default","id":"changefeed-error-1","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:57:44.295880376+08:00","start_ts":449546819774709761,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546819774709761,"checkpoint_ts":449546819774709761,"checkpoint_time":"2024-05-05 12:57:04.165"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/simple/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff failed 1-th time, retry later \033[0;36m<<< Run all test success >>>\033[0m table multi_tables_ddl_test.finish_mark exists check table exists success + endpoints=http://127.0.0.1:2379 + changefeed_id=test-normal + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s start tidb cluster in /tmp/tidb_cdc_test/owner_resign Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + info='{ "upstream_id": 7365375066399605070, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449546830411726861, "checkpoint_time": "2024-05-05 12:57:44.742", "error": null }' + echo '{ "upstream_id": 7365375066399605070, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449546830411726861, "checkpoint_time": "2024-05-05 12:57:44.742", "error": null }' { "upstream_id": 7365375066399605070, "namespace": "default", "id": "test-normal", "state": "normal", "checkpoint_tso": 449546830411726861, "checkpoint_time": "2024-05-05 12:57:44.742", "error": null } ++ echo '{' '"upstream_id":' 7365375066399605070, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449546830411726861, '"checkpoint_time":' '"2024-05-05' '12:57:44.742",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365375066399605070, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449546830411726861, '"checkpoint_time":' '"2024-05-05' '12:57:44.742",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-1 + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s + info='{ "upstream_id": 7365375066399605070, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449546830424834050, "checkpoint_time": "2024-05-05 12:57:44.792", "error": null }' + echo '{ "upstream_id": 7365375066399605070, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449546830424834050, "checkpoint_time": "2024-05-05 12:57:44.792", "error": null }' { "upstream_id": 7365375066399605070, "namespace": "default", "id": "test-error-1", "state": "normal", "checkpoint_tso": 449546830424834050, "checkpoint_time": "2024-05-05 12:57:44.792", "error": null } ++ echo '{' '"upstream_id":' 7365375066399605070, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449546830424834050, '"checkpoint_time":' '"2024-05-05' '12:57:44.792",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365375066399605070, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449546830424834050, '"checkpoint_time":' '"2024-05-05' '12:57:44.792",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + endpoints=http://127.0.0.1:2379 + changefeed_id=test-error-2 + expected_state=failed + error_msg=ErrSyncRenameTableFailed + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s find: '/tmp/tidb_cdc_test/*/*': No such file or directory =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/capture_session_done_during_task/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 30-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:45 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 {"id":"7d551d62-4d6f-4e1c-bc54-1ee59d600fa8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885062} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247642 7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 /tidb/cdc/default/default/upstream/7365375177104968028 {"id":7365375177104968028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 {"id":"7d551d62-4d6f-4e1c-bc54-1ee59d600fa8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885062} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247642 7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 /tidb/cdc/default/default/upstream/7365375177104968028 {"id":7365375177104968028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 {"id":"7d551d62-4d6f-4e1c-bc54-1ee59d600fa8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885062} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247642 7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 /tidb/cdc/default/default/upstream/7365375177104968028 {"id":7365375177104968028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun May 5 12:57:45 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.88238825.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 start tidb cluster in /tmp/tidb_cdc_test/batch_add_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + info='{ "upstream_id": 7365375066399605070, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449546822298370099, "checkpoint_time": "2024-05-05 12:57:13.792", "error": { "time": "2024-05-05T12:57:15.823079776+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' + echo '{ "upstream_id": 7365375066399605070, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449546822298370099, "checkpoint_time": "2024-05-05 12:57:13.792", "error": { "time": "2024-05-05T12:57:15.823079776+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } }' { "upstream_id": 7365375066399605070, "namespace": "default", "id": "test-error-2", "state": "failed", "checkpoint_tso": 449546822298370099, "checkpoint_time": "2024-05-05 12:57:13.792", "error": { "time": "2024-05-05T12:57:15.823079776+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSyncRenameTableFailed", "message": "[CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule." } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365375066399605070, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449546822298370099, '"checkpoint_time":' '"2024-05-05' '12:57:13.792",' '"error":' '{' '"time":' '"2024-05-05T12:57:15.823079776+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365375066399605070, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449546822298370099, '"checkpoint_time":' '"2024-05-05' '12:57:13.792",' '"error":' '{' '"time":' '"2024-05-05T12:57:15.823079776+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSyncRenameTableFailed",' '"message":' '"[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''130'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}' + message='[CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''130'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.' + [[ ! [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '130', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]] + set +x check_changefeed_status 127.0.0.1:8300 changefeed-error-1 warning last_warning ErrExecDDLFailed + endpoint=127.0.0.1:8300 + changefeed_id=changefeed-error-1 + expected_state=warning + field=last_warning + error_pattern=ErrExecDDLFailed ++ curl 127.0.0.1:8300/api/v2/changefeeds/changefeed-error-1/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully Verifying downstream PD is started... 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 253 100 253 0 0 1423 0 --:--:-- --:--:-- --:--:-- 1429 + info='{"state":"warning","resolved_ts":449546820154818585,"checkpoint_ts":449546820154818585,"last_warning":{"time":"2024-05-05T12:57:45.836923387+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' + echo '{"state":"warning","resolved_ts":449546820154818585,"checkpoint_ts":449546820154818585,"last_warning":{"time":"2024-05-05T12:57:45.836923387+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}}' {"state":"warning","resolved_ts":449546820154818585,"checkpoint_ts":449546820154818585,"last_warning":{"time":"2024-05-05T12:57:45.836923387+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec DDL failed"}} ++ echo '{"state":"warning","resolved_ts":449546820154818585,"checkpoint_ts":449546820154818585,"last_warning":{"time":"2024-05-05T12:57:45.836923387+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' ++ jq -r .state + state=warning + [[ ! warning == \w\a\r\n\i\n\g ]] + [[ -z last_warning ]] ++ jq -r .last_warning.message ++ echo '{"state":"warning","resolved_ts":449546820154818585,"checkpoint_ts":449546820154818585,"last_warning":{"time":"2024-05-05T12:57:45.836923387+08:00","addr":"127.0.0.1:8300","code":"CDC:ErrExecDDLFailed","message":"[CDC:ErrExecDDLFailed]exec' DDL 'failed"}}' + error_msg='[CDC:ErrExecDDLFailed]exec DDL failed' + [[ ! [CDC:ErrExecDDLFailed]exec DDL failed =~ ErrExecDDLFailed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6103.out cli changefeed remove -c changefeed-error-1 start tidb cluster in /tmp/tidb_cdc_test/resolve_lock Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/processor_resolved_ts_fallback Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 2-th time, retry later wait process cdc.test exit for 2-th time... start tidb cluster in /tmp/tidb_cdc_test/canal_json_storage_partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Changefeed remove successfully. ID: changefeed-error-1 CheckpointTs: 449546820154818585 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) wait process cdc.test exit for 3-th time... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:57:47 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>> Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/changefeed_finish Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + set +x start tidb cluster in /tmp/tidb_cdc_test/simple Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/capture_session_done_during_task Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... wait process cdc.test exit for 1-th time... check diff failed 3-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 31-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:49 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c5eb038-aad7-4be7-bf31-6e6479945ea2 {"id":"1c5eb038-aad7-4be7-bf31-6e6479945ea2","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885065} /tidb/cdc/default/__cdc_meta__/capture/7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 {"id":"7d551d62-4d6f-4e1c-bc54-1ee59d600fa8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885062} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247642 7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247676 1c5eb038-aad7-4be7-bf31-6e6479945ea2 /tidb/cdc/default/default/upstream/7365375177104968028 {"id":7365375177104968028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c5eb038-aad7-4be7-bf31-6e6479945ea2 {"id":"1c5eb038-aad7-4be7-bf31-6e6479945ea2","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885065} /tidb/cdc/default/__cdc_meta__/capture/7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 {"id":"7d551d62-4d6f-4e1c-bc54-1ee59d600fa8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885062} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247642 7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247676 1c5eb038-aad7-4be7-bf31-6e6479945ea2 /tidb/cdc/default/default/upstream/7365375177104968028 {"id":7365375177104968028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c5eb038-aad7-4be7-bf31-6e6479945ea2 {"id":"1c5eb038-aad7-4be7-bf31-6e6479945ea2","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885065} /tidb/cdc/default/__cdc_meta__/capture/7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 {"id":"7d551d62-4d6f-4e1c-bc54-1ee59d600fa8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885062} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247642 7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247676 1c5eb038-aad7-4be7-bf31-6e6479945ea2 /tidb/cdc/default/default/upstream/7365375177104968028 {"id":7365375177104968028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun May 5 12:57:49 CST 2024] <<<<<< START cdc server in multi_capture case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.88798881.out server --log-file /tmp/tidb_cdc_test/multi_capture/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_capture/cdc_data3 --cluster-id default --addr 127.0.0.1:8303 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8303; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... process cdc.test already exit check_etcd_meta_not_exist '/tidb/cdc/default/__cdc_meta__/owner' 'owner' + key_prefix=/tidb/cdc/default/__cdc_meta__/owner + message=owner ++ etcdctl get /tidb/cdc/default/__cdc_meta__/owner --prefix --keys-only + info= + [[ '' =~ owner ]] + echo 'check pass' check pass + exit 0 run task successfully [Sun May 5 12:57:49 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/txnutil/gc/InjectActualGCSafePoint=return(9223372036854775807)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.61706172.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Verifying downstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 32-th check, retry later check diff failed 4-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8303/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8303 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8303 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8303 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:52 GMT < Content-Length: 1756 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c5eb038-aad7-4be7-bf31-6e6479945ea2 {"id":"1c5eb038-aad7-4be7-bf31-6e6479945ea2","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885065} /tidb/cdc/default/__cdc_meta__/capture/7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 {"id":"7d551d62-4d6f-4e1c-bc54-1ee59d600fa8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885062} /tidb/cdc/default/__cdc_meta__/capture/dab3e8a4-80b3-4424-a735-b34ea9d01dbc {"id":"dab3e8a4-80b3-4424-a735-b34ea9d01dbc","address":"127.0.0.1:8303","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885069} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247642 7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247676 1c5eb038-aad7-4be7-bf31-6e6479945ea2 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247696 dab3e8a4-80b3-4424-a735-b34ea9d01dbc /tidb/cdc/default/default/upstream/7365375177104968028 {"id":7365375177104968028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c5eb038-aad7-4be7-bf31-6e6479945ea2 {"id":"1c5eb038-aad7-4be7-bf31-6e6479945ea2","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885065} /tidb/cdc/default/__cdc_meta__/capture/7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 {"id":"7d551d62-4d6f-4e1c-bc54-1ee59d600fa8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885062} /tidb/cdc/default/__cdc_meta__/capture/dab3e8a4-80b3-4424-a735-b34ea9d01dbc {"id":"dab3e8a4-80b3-4424-a735-b34ea9d01dbc","address":"127.0.0.1:8303","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885069} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247642 7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247676 1c5eb038-aad7-4be7-bf31-6e6479945ea2 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247696 dab3e8a4-80b3-4424-a735-b34ea9d01dbc /tidb/cdc/default/default/upstream/7365375177104968028 {"id":7365375177104968028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1c5eb038-aad7-4be7-bf31-6e6479945ea2 {"id":"1c5eb038-aad7-4be7-bf31-6e6479945ea2","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885065} /tidb/cdc/default/__cdc_meta__/capture/7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 {"id":"7d551d62-4d6f-4e1c-bc54-1ee59d600fa8","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885062} /tidb/cdc/default/__cdc_meta__/capture/dab3e8a4-80b3-4424-a735-b34ea9d01dbc {"id":"dab3e8a4-80b3-4424-a735-b34ea9d01dbc","address":"127.0.0.1:8303","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885069} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247642 7d551d62-4d6f-4e1c-bc54-1ee59d600fa8 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247676 1c5eb038-aad7-4be7-bf31-6e6479945ea2 /tidb/cdc/default/__cdc_meta__/owner/22318f471d247696 dab3e8a4-80b3-4424-a735-b34ea9d01dbc /tidb/cdc/default/default/upstream/7365375177104968028 {"id":7365375177104968028,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_capture.cli.8935.out cli changefeed create --start-ts=449546829200883713 '--sink-uri=file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-23117?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Create changefeed successfully! ID: ec04116a-83f0-4399-a9cd-af476ed0411b Info: {"upstream_id":7365375177104968028,"namespace":"default","id":"ec04116a-83f0-4399-a9cd-af476ed0411b","sink_uri":"file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-23117?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:57:52.615469834+08:00","start_ts":449546829200883713,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546829200883713,"checkpoint_ts":449546829200883713,"checkpoint_time":"2024-05-05 12:57:40.123"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 5-th time, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:57:53 GMT < Content-Length: 885 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e640e046-0557-4133-b5f2-6167a509c3c3 {"id":"e640e046-0557-4133-b5f2-6167a509c3c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885070} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e10c3 e640e046-0557-4133-b5f2-6167a509c3c3 /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e640e046-0557-4133-b5f2-6167a509c3c3 {"id":"e640e046-0557-4133-b5f2-6167a509c3c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885070} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e10c3 e640e046-0557-4133-b5f2-6167a509c3c3 /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/e640e046-0557-4133-b5f2-6167a509c3c3 {"id":"e640e046-0557-4133-b5f2-6167a509c3c3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885070} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 5 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e10c3 e640e046-0557-4133-b5f2-6167a509c3c3 /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6222.out cli changefeed create --start-ts=449546819774709761 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true' -c changefeed-error-2 table ddl_manager.finish_mark not exists for 33-th check, retry later Create changefeed successfully! ID: changefeed-error-2 Info: {"upstream_id":7365375037484554602,"namespace":"default","id":"changefeed-error-2","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:57:53.416513592+08:00","start_ts":449546819774709761,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546819774709761,"checkpoint_ts":449546819774709761,"checkpoint_time":"2024-05-05 12:57:04.165"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + workdir=/tmp/tidb_cdc_test/multi_capture + sink_uri='file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-23117?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/multi_capture ++ date + echo '[Sun May 5 12:57:54 CST 2024] <<<<<< START storage consumer in multi_capture case >>>>>>' [Sun May 5 12:57:54 CST 2024] <<<<<< START storage consumer in multi_capture case >>>>>> + cd /tmp/tidb_cdc_test/multi_capture + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/multi_capture + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/multi_capture/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/multi_capture/storage_test/ticdc-multi-capture-test-23117?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table multi_capture_1.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76624c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt, pid:1184, start at 2024-05-05 12:57:53.074561462 +0800 CST m=+5.170656238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:53.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:53.043 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:53.043 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76624c0014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt, pid:1184, start at 2024-05-05 12:57:53.074561462 +0800 CST m=+5.170656238 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:53.081 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:53.043 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:53.043 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7663c80014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt, pid:1257, start at 2024-05-05 12:57:53.177374225 +0800 CST m=+5.209049818 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:53.186 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:53.188 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:53.188 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/tiflash/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/tiflash/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/tiflash/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-error-2 failed [CDC:ErrSnapshotLostByGC] + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-error-2 + expected_state=failed + error_msg='[CDC:ErrSnapshotLostByGC]' + tls_dir='[CDC:ErrSnapshotLostByGC]' + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-error-2 -s Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + info='{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449546819774709761, "checkpoint_time": "2024-05-05 12:57:04.165", "error": { "time": "2024-05-05T12:57:53.508259919+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449546819774709761 is earlier than or equal to GC safepoint at 9223372036854775807" } }' + echo '{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449546819774709761, "checkpoint_time": "2024-05-05 12:57:04.165", "error": { "time": "2024-05-05T12:57:53.508259919+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449546819774709761 is earlier than or equal to GC safepoint at 9223372036854775807" } }' { "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-error-2", "state": "failed", "checkpoint_tso": 449546819774709761, "checkpoint_time": "2024-05-05 12:57:04.165", "error": { "time": "2024-05-05T12:57:53.508259919+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrSnapshotLostByGC", "message": "[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449546819774709761 is earlier than or equal to GC safepoint at 9223372036854775807" } } ++ jq -r .state ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449546819774709761, '"checkpoint_time":' '"2024-05-05' '12:57:04.165",' '"error":' '{' '"time":' '"2024-05-05T12:57:53.508259919+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449546819774709761 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + state=failed + [[ ! failed == \f\a\i\l\e\d ]] ++ jq -r .error.message ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449546819774709761, '"checkpoint_time":' '"2024-05-05' '12:57:04.165",' '"error":' '{' '"time":' '"2024-05-05T12:57:53.508259919+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrSnapshotLostByGC",' '"message":' '"[CDC:ErrSnapshotLostByGC]fail' to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449546819774709761 is earlier than or equal to GC safepoint at '9223372036854775807"' '}' '}' + message='[CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449546819774709761 is earlier than or equal to GC safepoint at 9223372036854775807' + [[ ! [CDC:ErrSnapshotLostByGC]fail to create or maintain changefeed due to snapshot loss caused by GC. checkpoint-ts 449546819774709761 is earlier than or equal to GC safepoint at 9223372036854775807 =~ \[CDC:ErrSnapshotLostByGC] ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6295.out cli changefeed remove -c changefeed-error-2 table ddl_manager.finish_mark not exists for 34-th check, retry later check diff failed 6-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76792c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc, pid:1296, start at 2024-05-05 12:57:54.553399333 +0800 CST m=+5.320376512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:54.560 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:54.558 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:54.558 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_capture_1.usertable not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Changefeed remove successfully. ID: changefeed-error-2 CheckpointTs: 449546819774709761 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.cli.2623.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7684a0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44, pid:1265, start at 2024-05-05 12:57:55.25643101 +0800 CST m=+5.128786215 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:55.262 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:55.240 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:55.240 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7684a0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44, pid:1265, start at 2024-05-05 12:57:55.25643101 +0800 CST m=+5.128786215 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:55.262 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:55.240 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:55.240 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7686140014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44, pid:1351, start at 2024-05-05 12:57:55.362975476 +0800 CST m=+5.179713417 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:55.371 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:55.333 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:55.333 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/batch_add_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/batch_add_table/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 35-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76792c0016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc, pid:1296, start at 2024-05-05 12:57:54.553399333 +0800 CST m=+5.320376512 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:54.560 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:54.558 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:54.558 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7685a00016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc, pid:1382, start at 2024-05-05 12:57:55.348169552 +0800 CST m=+6.019809555 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:55.358 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:55.355 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:55.355 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... check diff failed 7-th time, retry later + set +x [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/owner_resign/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/owner_resign/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/owner_resign/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } wait process cdc.test exit for 1-th time... + set +x + tso='449546833505288193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546833505288193 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 12:57:58 CST 2024] <<<<<< START cdc server in tiflash case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tiflash.26692671.out server --log-file /tmp/tidb_cdc_test/tiflash/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tiflash/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76a3240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp, pid:1248, start at 2024-05-05 12:57:57.215334502 +0800 CST m=+5.138790215 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:57.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:57.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:57.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76a3240013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp, pid:1248, start at 2024-05-05 12:57:57.215334502 +0800 CST m=+5.138790215 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:57.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:57.193 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:57.193 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76a4ac0017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-gn6-nz0rp, pid:1333, start at 2024-05-05 12:57:57.310554948 +0800 CST m=+5.175528599 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:57.318 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:57.291 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:57.291 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_resolved_ts_fallback/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table multi_capture_1.usertable not exists for 3-th check, retry later [Pipeline] // cache VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76a758000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm, pid:1312, start at 2024-05-05 12:57:57.477085185 +0800 CST m=+5.222493854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:57.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:57.462 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:57.462 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76a758000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm, pid:1312, start at 2024-05-05 12:57:57.477085185 +0800 CST m=+5.222493854 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:57.483 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:57.462 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:57.462 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76a8c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm, pid:1391, start at 2024-05-05 12:57:57.574970087 +0800 CST m=+5.269402997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:57.583 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:57.552 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:57.552 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:57:58 CST 2024] <<<<<< START cdc server in changefeed_error case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/redo/ChangefeedNewRedoManagerError=2*return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.63346336.out server --log-file /tmp/tidb_cdc_test/changefeed_error/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_error/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 36-th check, retry later check diff failed 8-th time, retry later [Sun May 5 12:57:59 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.26242626.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Sun May 5 12:57:59 CST 2024] <<<<<< START cdc server in batch_add_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.27422744.out server --log-file /tmp/tidb_cdc_test/batch_add_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/batch_add_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76a7800006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:9263, start at 2024-05-05 12:57:57.482774998 +0800 CST m=+5.411378080 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:57.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:57.472 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:57.472 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76a7800006 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:9263, start at 2024-05-05 12:57:57.482774998 +0800 CST m=+5.411378080 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:57.490 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:57.472 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:57.472 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76a8fc000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-8bm-r83hc, pid:9333, start at 2024-05-05 12:57:57.5809527 +0800 CST m=+5.455386199 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:57.589 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:57.567 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:57.567 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table multi_capture_1.usertable not exists for 4-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/canal_json_storage_partition_table/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2704.out cli tso query --pd=http://127.0.0.1:2379 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76cda80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51, pid:1374, start at 2024-05-05 12:57:59.943883324 +0800 CST m=+5.301804329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:59.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:59.914 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:59.914 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76b0e4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw, pid:1232, start at 2024-05-05 12:57:58.084747272 +0800 CST m=+5.119950515 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:58.090 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:58.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:58.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76b0e4000c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw, pid:1232, start at 2024-05-05 12:57:58.084747272 +0800 CST m=+5.119950515 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:58.090 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:58.073 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:58.073 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76b3240014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw, pid:1322, start at 2024-05-05 12:57:58.258397676 +0800 CST m=+5.220465780 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:58.266 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:58.266 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:58.266 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_finish/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_finish/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 9-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:01 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6287ea83-68ea-469f-87bc-2396eb3e6896 {"id":"6287ea83-68ea-469f-87bc-2396eb3e6896","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7083d0 6287ea83-68ea-469f-87bc-2396eb3e6896 /tidb/cdc/default/default/upstream/7365375266522343520 {"id":7365375266522343520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6287ea83-68ea-469f-87bc-2396eb3e6896 {"id":"6287ea83-68ea-469f-87bc-2396eb3e6896","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7083d0 6287ea83-68ea-469f-87bc-2396eb3e6896 /tidb/cdc/default/default/upstream/7365375266522343520 {"id":7365375266522343520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6287ea83-68ea-469f-87bc-2396eb3e6896 {"id":"6287ea83-68ea-469f-87bc-2396eb3e6896","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885078} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7083d0 6287ea83-68ea-469f-87bc-2396eb3e6896 /tidb/cdc/default/default/upstream/7365375266522343520 {"id":7365375266522343520,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: 392255a1-40d3-41ec-a50e-d92dd1b942c5 Info: {"upstream_id":7365375266522343520,"namespace":"default","id":"392255a1-40d3-41ec-a50e-d92dd1b942c5","sink_uri":"file:///tmp/tidb_cdc_test/tiflash/storage_test/ticdc-tiflash-test-1089?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:01.280064213+08:00","start_ts":449546833505288193,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546833505288193,"checkpoint_ts":449546833505288193,"checkpoint_time":"2024-05-05 12:57:56.543"} + workdir=/tmp/tidb_cdc_test/tiflash + sink_uri='file:///tmp/tidb_cdc_test/tiflash/storage_test/ticdc-tiflash-test-1089?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/tiflash ++ date + echo '[Sun May 5 12:58:01 CST 2024] <<<<<< START storage consumer in tiflash case >>>>>>' [Sun May 5 12:58:01 CST 2024] <<<<<< START storage consumer in tiflash case >>>>>> + cd /tmp/tidb_cdc_test/tiflash + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/tiflash + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/tiflash/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/tiflash/storage_test/ticdc-tiflash-test-1089?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76c8fc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:1257, start at 2024-05-05 12:57:59.642189637 +0800 CST m=+5.036772338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:59.649 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:59.615 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:59.615 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76c8fc0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:1257, start at 2024-05-05 12:57:59.642189637 +0800 CST m=+5.036772338 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:59.649 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:59.615 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:59.615 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76cc000002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:1335, start at 2024-05-05 12:57:59.810159131 +0800 CST m=+5.152106997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:59.817 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:59.808 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:59.808 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/simple/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/simple/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/simple/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/simple/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/simple/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table cdc_tiflash_test.multi_data_type not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 37-th check, retry later [Sun May 5 12:58:01 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=return(true);github.com/pingcap/tiflow/cdc/sink/dmlsink/mq/dmlproducer/KafkaSinkAsyncSendError=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.25372558.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } [Pipeline] // cache + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:02 GMT < Content-Length: 885 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b0d69d7-b672-4e3a-9cc9-8f393b97045d {"id":"0b0d69d7-b672-4e3a-9cc9-8f393b97045d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e112c 0b0d69d7-b672-4e3a-9cc9-8f393b97045d /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b0d69d7-b672-4e3a-9cc9-8f393b97045d {"id":"0b0d69d7-b672-4e3a-9cc9-8f393b97045d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e112c 0b0d69d7-b672-4e3a-9cc9-8f393b97045d /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0b0d69d7-b672-4e3a-9cc9-8f393b97045d {"id":"0b0d69d7-b672-4e3a-9cc9-8f393b97045d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/meta/ticdc-delete-etcd-key-count 7 /tidb/cdc/default/__cdc_meta__/owner/22318f471c9e112c 0b0d69d7-b672-4e3a-9cc9-8f393b97045d /tidb/cdc/default/default/upstream/7365375037484554602 {"id":7365375037484554602,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6389.out cli changefeed create --start-ts=0 '--sink-uri=file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true' -c changefeed-initialize-error table multi_capture_1.usertable not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > [Pipeline] } Create changefeed successfully! ID: changefeed-initialize-error Info: {"upstream_id":7365375037484554602,"namespace":"default","id":"changefeed-initialize-error","sink_uri":"file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:02.433390992+08:00","start_ts":449546835031752705,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546835031752705,"checkpoint_ts":449546835031752705,"checkpoint_time":"2024-05-05 12:58:02.366"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + tso='449546834650595329 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546834650595329 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 12:58:02 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.27432745.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:02 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0364fd10-750b-4f47-8c70-5eabfdb07ba0 {"id":"0364fd10-750b-4f47-8c70-5eabfdb07ba0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7b4015 0364fd10-750b-4f47-8c70-5eabfdb07ba0 /tidb/cdc/default/default/upstream/7365375280474748000 {"id":7365375280474748000,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0364fd10-750b-4f47-8c70-5eabfdb07ba0 {"id":"0364fd10-750b-4f47-8c70-5eabfdb07ba0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7b4015 0364fd10-750b-4f47-8c70-5eabfdb07ba0 /tidb/cdc/default/default/upstream/7365375280474748000 {"id":7365375280474748000,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0364fd10-750b-4f47-8c70-5eabfdb07ba0 {"id":"0364fd10-750b-4f47-8c70-5eabfdb07ba0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7b4015 0364fd10-750b-4f47-8c70-5eabfdb07ba0 /tidb/cdc/default/default/upstream/7365375280474748000 {"id":7365375280474748000,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.batch_add_table.cli.2798.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-4005?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 3930b761-6e1f-4cd9-810c-9eedd2880abc Info: {"upstream_id":7365375280474748000,"namespace":"default","id":"3930b761-6e1f-4cd9-810c-9eedd2880abc","sink_uri":"file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-4005?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:02.87016497+08:00","start_ts":449546835142901762,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546835142901762,"checkpoint_ts":449546835142901762,"checkpoint_time":"2024-05-05 12:58:02.790"} PASS + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:02 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc8eef2c-c83b-459b-ab50-c9a0dab11721 {"id":"bc8eef2c-c83b-459b-ab50-c9a0dab11721","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d779bce bc8eef2c-c83b-459b-ab50-c9a0dab11721 /tidb/cdc/default/default/upstream/7365375270729748060 {"id":7365375270729748060,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc8eef2c-c83b-459b-ab50-c9a0dab11721 {"id":"bc8eef2c-c83b-459b-ab50-c9a0dab11721","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d779bce bc8eef2c-c83b-459b-ab50-c9a0dab11721 /tidb/cdc/default/default/upstream/7365375270729748060 {"id":7365375270729748060,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/bc8eef2c-c83b-459b-ab50-c9a0dab11721 {"id":"bc8eef2c-c83b-459b-ab50-c9a0dab11721","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d779bce bc8eef2c-c83b-459b-ab50-c9a0dab11721 /tidb/cdc/default/default/upstream/7365375270729748060 {"id":7365375270729748060,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x cdc cli capture list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.cli.2721.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 [Sun May 5 12:58:02 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.1066810670.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc0 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sun May 5 12:58:02 CST 2024] <<<<<< START cdc server in changefeed_finish case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_finish.26352637.out server --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_finish/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 coverage: 2.5% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76cda80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51, pid:1374, start at 2024-05-05 12:57:59.943883324 +0800 CST m=+5.301804329 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-12:59:59.951 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:57:59.914 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:47:59.914 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c76ce5c0015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51, pid:1447, start at 2024-05-05 12:57:59.998639024 +0800 CST m=+5.271126351 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:00.004 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:00.008 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:00.008 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/capture_session_done_during_task/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Create changefeed successfully! ID: 3a476e71-db0b-4ab5-acd8-1cd706bf728c Info: {"upstream_id":7365375270729748060,"namespace":"default","id":"3a476e71-db0b-4ab5-acd8-1cd706bf728c","sink_uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:03.239889058+08:00","start_ts":449546835239370755,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546835239370755,"checkpoint_ts":449546835239370755,"checkpoint_time":"2024-05-05 12:58:03.158"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2695.out cli tso query --pd=http://127.0.0.1:2379 table cdc_tiflash_test.multi_data_type not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 38-th check, retry later check diff successfully [Pipeline] // dir [Pipeline] } + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s [Pipeline] // dir + info='{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449546835031752705, "checkpoint_time": "2024-05-05 12:58:02.366", "error": { "time": "2024-05-05T12:58:02.623649081+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449546835031752705, "checkpoint_time": "2024-05-05 12:58:02.366", "error": { "time": "2024-05-05T12:58:02.623649081+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "warning", "checkpoint_tso": 449546835031752705, "checkpoint_time": "2024-05-05 12:58:02.366", "error": { "time": "2024-05-05T12:58:02.623649081+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"warning",' '"checkpoint_tso":' 449546835031752705, '"checkpoint_time":' '"2024-05-05' '12:58:02.366",' '"error":' '{' '"time":' '"2024-05-05T12:58:02.623649081+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=warning + [[ ! warning == \n\o\r\m\a\l ]] + echo 'changefeed state warning does not equal to normal' changefeed state warning does not equal to normal + exit 1 run task failed 1-th time, retry later table multi_capture_1.usertable not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:04 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b810204-8b8e-4ccf-91de-a7b26554b616 {"id":"6b810204-8b8e-4ccf-91de-a7b26554b616","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885081} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7cd2d1 6b810204-8b8e-4ccf-91de-a7b26554b616 /tidb/cdc/default/default/upstream/7365375277098550907 {"id":7365375277098550907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b810204-8b8e-4ccf-91de-a7b26554b616 {"id":"6b810204-8b8e-4ccf-91de-a7b26554b616","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885081} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7cd2d1 6b810204-8b8e-4ccf-91de-a7b26554b616 /tidb/cdc/default/default/upstream/7365375277098550907 {"id":7365375277098550907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6b810204-8b8e-4ccf-91de-a7b26554b616 {"id":"6b810204-8b8e-4ccf-91de-a7b26554b616","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885081} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7cd2d1 6b810204-8b8e-4ccf-91de-a7b26554b616 /tidb/cdc/default/default/upstream/7365375277098550907 {"id":7365375277098550907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.cli.2703.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8301 + set +x + workdir=/tmp/tidb_cdc_test/batch_add_table + sink_uri='file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-4005?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/batch_add_table ++ date + echo '[Sun May 5 12:58:04 CST 2024] <<<<<< START storage consumer in batch_add_table case >>>>>>' [Sun May 5 12:58:04 CST 2024] <<<<<< START storage consumer in batch_add_table case >>>>>> + cd /tmp/tidb_cdc_test/batch_add_table + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/batch_add_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/batch_add_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/batch_add_table/storage_test/ticdc-batch-add-table-test-4005?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table batch_add_table.finish_mark not exists for 1-th check, retry later [Pipeline] } [Pipeline] // withCredentials [Pipeline] } Create changefeed successfully! ID: 8bda9019-0498-47f2-8759-e93affd9c0c3 Info: {"upstream_id":7365375277098550907,"namespace":"default","id":"8bda9019-0498-47f2-8759-e93affd9c0c3","sink_uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:04.66179173+08:00","start_ts":449546835602178053,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546835602178053,"checkpoint_ts":449546835602178053,"checkpoint_time":"2024-05-05 12:58:04.542"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + workdir=/tmp/tidb_cdc_test/owner_resign + sink_uri='file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/owner_resign ++ date + echo '[Sun May 5 12:58:04 CST 2024] <<<<<< START storage consumer in owner_resign case >>>>>>' [Sun May 5 12:58:04 CST 2024] <<<<<< START storage consumer in owner_resign case >>>>>> + cd /tmp/tidb_cdc_test/owner_resign + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/owner_resign + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/owner_resign/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' 1 run task successfully table owner_resign.t1 not exists for 1-th check, retry later [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // stage [Pipeline] } + set +x + tso='449546835385384961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546835385384961 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 39-th check, retry later [Pipeline] // container [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:05 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa7f932b-5d0c-4777-b1b4-e466e016f924 {"id":"fa7f932b-5d0c-4777-b1b4-e466e016f924","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d7cd fa7f932b-5d0c-4777-b1b4-e466e016f924 /tidb/cdc/default/default/upstream/7365375287312128763 {"id":7365375287312128763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa7f932b-5d0c-4777-b1b4-e466e016f924 {"id":"fa7f932b-5d0c-4777-b1b4-e466e016f924","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d7cd fa7f932b-5d0c-4777-b1b4-e466e016f924 /tidb/cdc/default/default/upstream/7365375287312128763 {"id":7365375287312128763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fa7f932b-5d0c-4777-b1b4-e466e016f924 {"id":"fa7f932b-5d0c-4777-b1b4-e466e016f924","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d7cd fa7f932b-5d0c-4777-b1b4-e466e016f924 /tidb/cdc/default/default/upstream/7365375287312128763 {"id":7365375287312128763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun May 5 12:58:05 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.1072210724.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // container [Pipeline] } check diff failed 1-th time, retry later [Pipeline] // withEnv + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] } + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.cli.2733.out cli tso query --pd=http://127.0.0.1:2379 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:05 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/550dc1d1-4a8c-41cd-b529-90ef0900f3ab {"id":"550dc1d1-4a8c-41cd-b529-90ef0900f3ab","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7dadd0 550dc1d1-4a8c-41cd-b529-90ef0900f3ab /tidb/cdc/default/default/upstream/7365375291974706174 {"id":7365375291974706174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/550dc1d1-4a8c-41cd-b529-90ef0900f3ab {"id":"550dc1d1-4a8c-41cd-b529-90ef0900f3ab","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7dadd0 550dc1d1-4a8c-41cd-b529-90ef0900f3ab /tidb/cdc/default/default/upstream/7365375291974706174 {"id":7365375291974706174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/550dc1d1-4a8c-41cd-b529-90ef0900f3ab {"id":"550dc1d1-4a8c-41cd-b529-90ef0900f3ab","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7dadd0 550dc1d1-4a8c-41cd-b529-90ef0900f3ab /tidb/cdc/default/default/upstream/7365375291974706174 {"id":7365375291974706174,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2800.out cli changefeed create --start-ts=449546834650595329 '--sink-uri=file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-13427?protocol=canal-json&enable-tidb-extension=true' [Pipeline] // withEnv [Pipeline] } Create changefeed successfully! ID: 15d5385d-a2bf-445e-8991-4a1f62941bc5 Info: {"upstream_id":7365375291974706174,"namespace":"default","id":"15d5385d-a2bf-445e-8991-4a1f62941bc5","sink_uri":"file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-13427?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:05.851042435+08:00","start_ts":449546834650595329,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546834650595329,"checkpoint_ts":449546834650595329,"checkpoint_time":"2024-05-05 12:58:00.912"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Pipeline] // node table cdc_tiflash_test.multi_data_type not exists for 3-th check, retry later [Pipeline] } + set +x + workdir=/tmp/tidb_cdc_test/processor_resolved_ts_fallback + sink_uri='file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_resolved_ts_fallback ++ date + echo '[Sun May 5 12:58:06 CST 2024] <<<<<< START storage consumer in processor_resolved_ts_fallback case >>>>>>' [Sun May 5 12:58:06 CST 2024] <<<<<< START storage consumer in processor_resolved_ts_fallback case >>>>>> + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_resolved_ts_fallback + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' [Pipeline] // node [Pipeline] } [Sun May 5 12:58:06 CST 2024] <<<<<< START cdc server in simple case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.27502752.out server --log-file /tmp/tidb_cdc_test/simple/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/simple/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:05 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5e6eb95-4e94-4d99-83d6-e5290f3c03fe {"id":"a5e6eb95-4e94-4d99-83d6-e5290f3c03fe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885083} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d8269cf a5e6eb95-4e94-4d99-83d6-e5290f3c03fe /tidb/cdc/default/default/upstream/7365375290370071021 {"id":7365375290370071021,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5e6eb95-4e94-4d99-83d6-e5290f3c03fe {"id":"a5e6eb95-4e94-4d99-83d6-e5290f3c03fe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885083} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d8269cf a5e6eb95-4e94-4d99-83d6-e5290f3c03fe /tidb/cdc/default/default/upstream/7365375290370071021 {"id":7365375290370071021,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/a5e6eb95-4e94-4d99-83d6-e5290f3c03fe {"id":"a5e6eb95-4e94-4d99-83d6-e5290f3c03fe","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885083} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d8269cf a5e6eb95-4e94-4d99-83d6-e5290f3c03fe /tidb/cdc/default/default/upstream/7365375290370071021 {"id":7365375290370071021,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/changefeed_finish + sink_uri='file:///tmp/tidb_cdc_test/changefeed_finish/storage_test/ticdc-changefeed-pause-resume-32682?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_finish ++ date + echo '[Sun May 5 12:58:06 CST 2024] <<<<<< START storage consumer in changefeed_finish case >>>>>>' [Sun May 5 12:58:06 CST 2024] <<<<<< START storage consumer in changefeed_finish case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_finish + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_finish + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_finish/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_finish/storage_test/ticdc-changefeed-pause-resume-32682?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' [Pipeline] // podTemplate [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449546835883720708, "checkpoint_time": "2024-05-05 12:58:05.616", "error": null }' + echo '{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449546835883720708, "checkpoint_time": "2024-05-05 12:58:05.616", "error": null }' { "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449546835883720708, "checkpoint_time": "2024-05-05 12:58:05.616", "error": null } ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449546835883720708, '"checkpoint_time":' '"2024-05-05' '12:58:05.616",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449546835883720708, '"checkpoint_time":' '"2024-05-05' '12:58:05.616",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6541.out cli changefeed pause -c changefeed-initialize-error [Pipeline] } table multi_capture_1.usertable exists table multi_capture_2.usertable exists table multi_capture_3.usertable exists table multi_capture_4.usertable not exists for 1-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... check diff failed 1-th time, retry later [Pipeline] // stage table batch_add_table.finish_mark not exists for 2-th check, retry later [Pipeline] } cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^1$' + set +x + tso='449546835909148676 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546835909148676 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table ddl_manager.finish_mark not exists for 40-th check, retry later 1 run task successfully table processor_resolved_ts_fallback.t1 not exists for 1-th check, retry later [Sun May 5 12:58:07 CST 2024] <<<<<< START cdc server in capture_session_done_during_task case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorManagerHandleNewChangefeedDelay=sleep(2000)' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.capture_session_done_during_task.27762778.out server --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + workdir=/tmp/tidb_cdc_test/resolve_lock + sink_uri='file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-13427?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/resolve_lock ++ date + echo '[Sun May 5 12:58:07 CST 2024] <<<<<< START storage consumer in resolve_lock case >>>>>>' [Sun May 5 12:58:07 CST 2024] <<<<<< START storage consumer in resolve_lock case >>>>>> + cd /tmp/tidb_cdc_test/resolve_lock + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/resolve_lock + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/resolve_lock/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/resolve_lock/storage_test/ticdc-resolve-lock-test-13427?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' go: downloading github.com/pingcap/kvproto v0.0.0-20240227073058-929ab83f9754 go: downloading github.com/pingcap/log v1.1.1-0.20240314023424-862ccc32f18d go: downloading github.com/pingcap/tidb v1.1.0-beta.0.20240415145106-cd9c676e9ba4 go: downloading github.com/tikv/client-go/v2 v2.0.8-0.20240409022718-714958ccd4d5 go: downloading github.com/pingcap/tidb/pkg/parser v0.0.0-20240410110152-5fc42c9be2f5 go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f go: downloading github.com/tikv/pd/client v0.0.0-20240322051414-fb9e2d561b6e go: downloading github.com/BurntSushi/toml v1.3.2 go: downloading go.uber.org/zap v1.27.0 go: downloading github.com/pingcap/tidb-tools v0.0.0-20240305021104-9f9bea84490b go: downloading gopkg.in/natefinch/lumberjack.v2 v2.2.1 go: downloading go.uber.org/atomic v1.11.0 check diff failed 2-th time, retry later table owner_resign.t1 not exists for 2-th check, retry later go: downloading github.com/tikv/pd v1.1.0-beta.0.20240407022249-7179657d129b go: downloading go.uber.org/multierr v1.11.0 go: downloading github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c go: downloading google.golang.org/grpc v1.62.1 go: downloading github.com/coreos/go-semver v0.3.1 go: downloading github.com/go-sql-driver/mysql v1.7.1 go: downloading github.com/google/uuid v1.6.0 go: downloading github.com/pkg/errors v0.9.1 go: downloading github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a go: downloading go.etcd.io/etcd/client/v3 v3.5.12 go: downloading go.etcd.io/etcd/api/v3 v3.5.12 go: downloading github.com/golang/protobuf v1.5.4 go: downloading github.com/opentracing/opentracing-go v1.2.0 go: downloading github.com/prometheus/client_golang v1.19.0 go: downloading github.com/prometheus/client_model v0.6.1 go: downloading github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 go: downloading github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 go: downloading github.com/twmb/murmur3 v1.1.6 go: downloading github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 go: downloading github.com/gogo/protobuf v1.3.2 go: downloading github.com/docker/go-units v0.5.0 go: downloading github.com/google/btree v1.1.2 go: downloading golang.org/x/sync v0.7.0 go: downloading google.golang.org/protobuf v1.33.0 go: downloading github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec go: downloading go.etcd.io/etcd/client/pkg/v3 v3.5.12 go: downloading google.golang.org/genproto/googleapis/api v0.0.0-20240401170217-c3f982113cda go: downloading github.com/coreos/go-systemd/v22 v22.5.0 go: downloading github.com/cespare/xxhash/v2 v2.3.0 go: downloading github.com/prometheus/procfs v0.13.0 go: downloading github.com/prometheus/common v0.52.2 go: downloading github.com/beorn7/perks v1.0.1 go: downloading google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda table cdc_tiflash_test.multi_data_type not exists for 4-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/csv_storage_partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... go: downloading golang.org/x/sys v0.19.0 go: downloading golang.org/x/exp v0.0.0-20240409090435-93d18d7e34b8 go: downloading github.com/cloudfoundry/gosigar v1.3.6 + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error stopped changefeed new redo manager injected error + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=stopped + error_msg=changefeed + tls_dir=error + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s go: downloading golang.org/x/net v0.24.0 go: downloading google.golang.org/genproto/googleapis/rpc v0.0.0-20240401170217-c3f982113cda + info='{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449546835883720711, "checkpoint_time": "2024-05-05 12:58:05.616", "error": { "time": "2024-05-05T12:58:02.623649081+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' + echo '{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449546835883720711, "checkpoint_time": "2024-05-05 12:58:05.616", "error": { "time": "2024-05-05T12:58:02.623649081+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } }' { "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "stopped", "checkpoint_tso": 449546835883720711, "checkpoint_time": "2024-05-05 12:58:05.616", "error": { "time": "2024-05-05T12:58:02.623649081+08:00", "addr": "127.0.0.1:8300", "code": "CDC:ErrProcessorUnknown", "message": "changefeed new redo manager injected error" } } ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449546835883720711, '"checkpoint_time":' '"2024-05-05' '12:58:05.616",' '"error":' '{' '"time":' '"2024-05-05T12:58:02.623649081+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"stopped",' '"checkpoint_tso":' 449546835883720711, '"checkpoint_time":' '"2024-05-05' '12:58:05.616",' '"error":' '{' '"time":' '"2024-05-05T12:58:02.623649081+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrProcessorUnknown",' '"message":' '"changefeed' new redo manager injected 'error"' '}' '}' ++ jq -r .error.message + message='changefeed new redo manager injected error' + [[ ! changefeed new redo manager injected error =~ changefeed ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6625.out cli changefeed resume -c changefeed-initialize-error table multi_capture_4.usertable not exists for 2-th check, retry later go: downloading golang.org/x/text v0.14.0 check diff failed 2-th time, retry later table batch_add_table.finish_mark not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:08 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf {"id":"8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885085} /tidb/cdc/default/__cdc_meta__/capture/fa7f932b-5d0c-4777-b1b4-e466e016f924 {"id":"fa7f932b-5d0c-4777-b1b4-e466e016f924","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d7cd fa7f932b-5d0c-4777-b1b4-e466e016f924 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d801 8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf /tidb/cdc/default/default/upstream/7365375287312128763 {"id":7365375287312128763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf {"id":"8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885085} /tidb/cdc/default/__cdc_meta__/capture/fa7f932b-5d0c-4777-b1b4-e466e016f924 {"id":"fa7f932b-5d0c-4777-b1b4-e466e016f924","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d7cd fa7f932b-5d0c-4777-b1b4-e466e016f924 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d801 8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf /tidb/cdc/default/default/upstream/7365375287312128763 {"id":7365375287312128763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf {"id":"8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885085} /tidb/cdc/default/__cdc_meta__/capture/fa7f932b-5d0c-4777-b1b4-e466e016f924 {"id":"fa7f932b-5d0c-4777-b1b4-e466e016f924","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d7cd fa7f932b-5d0c-4777-b1b4-e466e016f924 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d801 8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf /tidb/cdc/default/default/upstream/7365375287312128763 {"id":7365375287312128763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun May 5 12:58:08 CST 2024] <<<<<< START cdc server in canal_json_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.1077710779.out server --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdccdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_datacdc2 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 41-th check, retry later table processor_resolved_ts_fallback.t1 not exists for 2-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... table owner_resign.t1 not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:09 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53b85e3f-81dc-4303-ac29-46922efb918c {"id":"53b85e3f-81dc-4303-ac29-46922efb918c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885086} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d86def1 53b85e3f-81dc-4303-ac29-46922efb918c /tidb/cdc/default/default/upstream/7365375300350930276 {"id":7365375300350930276,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53b85e3f-81dc-4303-ac29-46922efb918c {"id":"53b85e3f-81dc-4303-ac29-46922efb918c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885086} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d86def1 53b85e3f-81dc-4303-ac29-46922efb918c /tidb/cdc/default/default/upstream/7365375300350930276 {"id":7365375300350930276,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/53b85e3f-81dc-4303-ac29-46922efb918c {"id":"53b85e3f-81dc-4303-ac29-46922efb918c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885086} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d86def1 53b85e3f-81dc-4303-ac29-46922efb918c /tidb/cdc/default/default/upstream/7365375300350930276 {"id":7365375300350930276,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.simple.cli.2813.out cli changefeed create --start-ts=449546835385384961 '--sink-uri=file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-28594?protocol=canal-json&enable-tidb-extension=true' table cdc_tiflash_test.multi_data_type not exists for 5-th check, retry later Create changefeed successfully! ID: 1a822b8c-c96c-4b72-a759-ea94ac327c58 Info: {"upstream_id":7365375300350930276,"namespace":"default","id":"1a822b8c-c96c-4b72-a759-ea94ac327c58","sink_uri":"file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-28594?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:09.699821721+08:00","start_ts":449546835385384961,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546835385384961,"checkpoint_ts":449546835385384961,"checkpoint_time":"2024-05-05 12:58:03.715"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c774ea0001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:6650, start at 2024-05-05 12:58:08.216194093 +0800 CST m=+5.208228891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:08.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:08.217 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:08.217 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c774ea0001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:6650, start at 2024-05-05 12:58:08.216194093 +0800 CST m=+5.208228891 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:08.222 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:08.217 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:08.217 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7750c8000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:6736, start at 2024-05-05 12:58:08.32097675 +0800 CST m=+5.255337752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:08.330 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:08.306 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:08.306 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 3-th time, retry later table multi_capture_4.usertable not exists for 3-th check, retry later go: downloading github.com/pingcap/tipb v0.0.0-20240318032315-55a7867ddd50 go: downloading github.com/opentracing/basictracer-go v1.1.0 go: downloading github.com/coocood/freecache v1.2.1 go: downloading github.com/pingcap/sysutil v1.0.1-0.20240311050922-ae81ee01f3a5 go: downloading github.com/uber/jaeger-client-go v2.30.0+incompatible go: downloading github.com/shirou/gopsutil/v3 v3.24.2 go: downloading github.com/dgraph-io/ristretto v0.1.1 go: downloading github.com/cockroachdb/errors v1.11.1 go: downloading github.com/spf13/pflag v1.0.5 go: downloading gopkg.in/yaml.v2 v2.4.0 go: downloading github.com/jellydator/ttlcache/v3 v3.0.1 go: downloading github.com/gorilla/mux v1.8.0 go: downloading github.com/influxdata/tdigest v0.0.1 go: downloading github.com/danjacques/gofslock v0.0.0-20240212154529-d899e02bfe22 go: downloading github.com/ngaut/pools v0.0.0-20180318154953-b7bc8c42aac7 go: downloading github.com/yangkeao/ldap/v3 v3.4.5-0.20230421065457-369a3bab1117 go: downloading github.com/dolthub/swiss v0.2.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 go: downloading cloud.google.com/go/storage v1.39.1 go: downloading github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 go: downloading github.com/golang/snappy v0.0.4 go: downloading github.com/Azure/azure-sdk-for-go/sdk/azcore v1.9.1 go: downloading github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 go: downloading github.com/aws/aws-sdk-go v1.50.0 go: downloading github.com/go-resty/resty/v2 v2.11.0 go: downloading github.com/klauspost/compress v1.17.8 go: downloading golang.org/x/tools v0.20.0 go: downloading github.com/tidwall/btree v1.7.0 go: downloading github.com/scalalang2/golang-fifo v0.1.5 go: downloading github.com/stretchr/testify v1.9.0 go: downloading github.com/ks3sdklib/aws-sdk-go v1.2.9 go: downloading go.uber.org/mock v0.4.0 go: downloading github.com/otiai10/copy v1.2.0 go: downloading github.com/lestrrat-go/jwx/v2 v2.0.21 go: downloading github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 go: downloading github.com/cockroachdb/pebble v1.1.0 go: downloading golang.org/x/oauth2 v0.18.0 go: downloading google.golang.org/api v0.170.0 go: downloading github.com/jfcg/sorty/v2 v2.1.0 go: downloading github.com/joho/sqltocsv v0.0.0-20210428211105-a6d6801d59df go: downloading github.com/cheggaaa/pb/v3 v3.0.8 go: downloading github.com/carlmjohnson/flagext v0.21.0 go: downloading github.com/spkg/bom v1.0.0 go: downloading github.com/xitongsys/parquet-go v1.6.0 go: downloading golang.org/x/time v0.5.0 go: downloading github.com/ngaut/sync2 v0.0.0-20141008032647-7a24ed77b2ef go: downloading github.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0 go: downloading github.com/jedib0t/go-pretty/v6 v6.2.2 go: downloading github.com/pingcap/badger v1.5.1-0.20230103063557-828f39b09b6d go: downloading github.com/dolthub/maphash v0.1.0 go: downloading github.com/pingcap/goleveldb v0.0.0-20191226122134-f82aafb29989 go: downloading github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 go: downloading github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 go: downloading github.com/go-asn1-ber/asn1-ber v1.5.4 go: downloading github.com/wangjohn/quickselect v0.0.0-20161129230411-ed8402a42d5f go: downloading github.com/robfig/cron/v3 v3.0.1 go: downloading github.com/VividCortex/ewma v1.2.0 go: downloading github.com/fatih/color v1.16.0 go: downloading github.com/mattn/go-colorable v0.1.13 go: downloading github.com/mattn/go-isatty v0.0.20 go: downloading github.com/mattn/go-runewidth v0.0.15 go: downloading github.com/vbauerster/mpb/v7 v7.5.3 go: downloading golang.org/x/term v0.19.0 go: downloading github.com/spf13/cobra v1.8.0 go: downloading github.com/json-iterator/go v1.1.12 go: downloading github.com/robfig/cron v1.2.0 go: downloading cloud.google.com/go v0.112.2 go: downloading github.com/jfcg/sixb v1.3.8 go: downloading github.com/Azure/azure-sdk-for-go/sdk/internal v1.5.1 go: downloading github.com/AzureAD/microsoft-authentication-library-for-go v1.2.1 go: downloading golang.org/x/crypto v0.22.0 go: downloading github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc go: downloading github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 go: downloading gopkg.in/yaml.v3 v3.0.1 go: downloading github.com/apache/thrift v0.16.0 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:10 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9736b7a3-11f5-4b54-9ad4-c3ff686e80b1 {"id":"9736b7a3-11f5-4b54-9ad4-c3ff686e80b1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885087} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d86dbf6 9736b7a3-11f5-4b54-9ad4-c3ff686e80b1 /tidb/cdc/default/default/upstream/7365375288696189188 {"id":7365375288696189188,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9736b7a3-11f5-4b54-9ad4-c3ff686e80b1 {"id":"9736b7a3-11f5-4b54-9ad4-c3ff686e80b1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885087} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d86dbf6 9736b7a3-11f5-4b54-9ad4-c3ff686e80b1 /tidb/cdc/default/default/upstream/7365375288696189188 {"id":7365375288696189188,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9736b7a3-11f5-4b54-9ad4-c3ff686e80b1 {"id":"9736b7a3-11f5-4b54-9ad4-c3ff686e80b1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885087} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d86dbf6 9736b7a3-11f5-4b54-9ad4-c3ff686e80b1 /tidb/cdc/default/default/upstream/7365375288696189188 {"id":7365375288696189188,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table batch_add_table.finish_mark not exists for 4-th check, retry later + set +x check_changefeed_state http://127.0.0.1:2379 changefeed-initialize-error normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=changefeed-initialize-error + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c changefeed-initialize-error -s + info='{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449546835883720711, "checkpoint_time": "2024-05-05 12:58:05.616", "error": null }' + echo '{ "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449546835883720711, "checkpoint_time": "2024-05-05 12:58:05.616", "error": null }' { "upstream_id": 7365375037484554602, "namespace": "default", "id": "changefeed-initialize-error", "state": "normal", "checkpoint_tso": 449546835883720711, "checkpoint_time": "2024-05-05 12:58:05.616", "error": null } ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449546835883720711, '"checkpoint_time":' '"2024-05-05' '12:58:05.616",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365375037484554602, '"namespace":' '"default",' '"id":' '"changefeed-initialize-error",' '"state":' '"normal",' '"checkpoint_tso":' 449546835883720711, '"checkpoint_time":' '"2024-05-05' '12:58:05.616",' '"error":' null '}' ++ jq -r .error.message go: downloading cloud.google.com/go/compute/metadata v0.2.3 go: downloading cloud.google.com/go/compute v1.25.1 go: downloading github.com/Masterminds/semver v1.5.0 go: downloading k8s.io/api v0.28.6 go: downloading github.com/emirpasic/gods v1.18.1 go: downloading github.com/rivo/uniseg v0.4.7 go: downloading github.com/dustin/go-humanize v1.0.1 go: downloading github.com/golang/glog v1.2.0 go: downloading github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b go: downloading github.com/cockroachdb/redact v1.1.5 go: downloading github.com/getsentry/sentry-go v0.27.0 go: downloading github.com/uber/jaeger-lib v2.4.1+incompatible go: downloading github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d go: downloading github.com/lestrrat-go/blackmagic v1.0.2 go: downloading github.com/lestrrat-go/httprc v1.0.5 go: downloading github.com/lestrrat-go/iter v1.0.2 go: downloading github.com/lestrrat-go/option v1.0.1 go: downloading github.com/kr/pretty v0.3.1 go: downloading github.com/modern-go/reflect2 v1.0.2 go: downloading github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd go: downloading github.com/tklauser/go-sysconf v0.3.12 go: downloading github.com/lestrrat-go/httpcc v1.0.1 go: downloading github.com/rogpeppe/go-internal v1.12.0 go: downloading github.com/kr/text v0.2.0 go: downloading github.com/ncw/directio v1.0.5 go: downloading github.com/coocood/rtutil v0.0.0-20190304133409-c84515f646f2 go: downloading github.com/coocood/bbloom v0.0.0-20190830030839-58deb6228d64 go: downloading github.com/klauspost/cpuid v1.3.1 go: downloading github.com/kylelemons/godebug v1.1.0 go: downloading github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c go: downloading github.com/golang-jwt/jwt/v5 v5.2.0 go: downloading github.com/tklauser/numcpus v0.6.1 go: downloading cloud.google.com/go/iam v1.1.7 go: downloading github.com/googleapis/gax-go/v2 v2.12.3 go: downloading go.opentelemetry.io/otel v1.24.0 go: downloading go.opentelemetry.io/otel/trace v1.24.0 go: downloading go.opencensus.io v0.23.1-0.20220331163232-052120675fac + message=null + [[ ! null =~ null ]] run task successfully + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_error.cli.6715.out cli changefeed remove -c changefeed-initialize-error go: downloading github.com/golang-jwt/jwt v3.2.2+incompatible + set +x + workdir=/tmp/tidb_cdc_test/simple + sink_uri='file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-28594?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/simple ++ date + echo '[Sun May 5 12:58:11 CST 2024] <<<<<< START storage consumer in simple case >>>>>>' [Sun May 5 12:58:11 CST 2024] <<<<<< START storage consumer in simple case >>>>>> + cd /tmp/tidb_cdc_test/simple + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/simple + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/simple/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/simple/storage_test/ticdc-simple-test-28594?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' succeed to verify meta placement rules check diff failed 3-th time, retry later table owner_resign.t1 not exists for 4-th check, retry later go: downloading github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da go: downloading go.opentelemetry.io/otel/metric v1.24.0 go: downloading github.com/go-logr/logr v1.4.1 go: downloading github.com/go-logr/stdr v1.2.2 ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 1-th time, retry later + workdir=/tmp/tidb_cdc_test/capture_session_done_during_task + sink_uri='file:///tmp/tidb_cdc_test/capture_session_done_during_task/storage_test/ticdc-capture-session-done-during-task-7223?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/capture_session_done_during_task ++ date + echo '[Sun May 5 12:58:11 CST 2024] <<<<<< START storage consumer in capture_session_done_during_task case >>>>>>' [Sun May 5 12:58:11 CST 2024] <<<<<< START storage consumer in capture_session_done_during_task case >>>>>> + cd /tmp/tidb_cdc_test/capture_session_done_during_task + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/capture_session_done_during_task + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/capture_session_done_during_task/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/capture_session_done_during_task/storage_test/ticdc-capture-session-done-during-task-7223?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' lease 22318f471d86dbf6 revoked table cdc_tiflash_test.multi_data_type exists table ddl_manager.finish_mark not exists for 42-th check, retry later table processor_resolved_ts_fallback.t1 not exists for 3-th check, retry later [Sun May 5 12:58:11 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.79927994.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 4-th time, retry later go: downloading github.com/DataDog/zstd v1.5.5 go: downloading github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 check diff successfully Changefeed remove successfully. ID: changefeed-initialize-error CheckpointTs: 449546837194440709 SinkURI: file:///tmp/tidb_cdc_test/changefeed_error/storage_test/ticdc-sink-retry-test-21918?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... go: downloading k8s.io/apimachinery v0.28.6 go: downloading gopkg.in/inf.v0 v0.9.1 go: downloading k8s.io/klog/v2 v2.120.1 go: downloading github.com/google/gofuzz v1.2.0 go: downloading sigs.k8s.io/structured-merge-diff/v4 v4.4.1 go: downloading sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd go: downloading k8s.io/utils v0.0.0-20230726121419-3b25d923346b + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:11 GMT < Content-Length: 1756 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf {"id":"8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885085} /tidb/cdc/default/__cdc_meta__/capture/9d40845a-d950-4202-9406-278a9b8b0f87 {"id":"9d40845a-d950-4202-9406-278a9b8b0f87","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885088} /tidb/cdc/default/__cdc_meta__/capture/fa7f932b-5d0c-4777-b1b4-e466e016f924 {"id":"fa7f932b-5d0c-4777-b1b4-e466e016f924","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d7cd fa7f932b-5d0c-4777-b1b4-e466e016f924 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d801 8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d81c 9d40845a-d950-4202-9406-278a9b8b0f87 /tidb/cdc/default/default/upstream/7365375287312128763 {"id":7365375287312128763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf {"id":"8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885085} /tidb/cdc/default/__cdc_meta__/capture/9d40845a-d950-4202-9406-278a9b8b0f87 {"id":"9d40845a-d950-4202-9406-278a9b8b0f87","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885088} /tidb/cdc/default/__cdc_meta__/capture/fa7f932b-5d0c-4777-b1b4-e466e016f924 {"id":"fa7f932b-5d0c-4777-b1b4-e466e016f924","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d7cd fa7f932b-5d0c-4777-b1b4-e466e016f924 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d801 8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d81c 9d40845a-d950-4202-9406-278a9b8b0f87 /tidb/cdc/default/default/upstream/7365375287312128763 {"id":7365375287312128763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf {"id":"8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885085} /tidb/cdc/default/__cdc_meta__/capture/9d40845a-d950-4202-9406-278a9b8b0f87 {"id":"9d40845a-d950-4202-9406-278a9b8b0f87","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885088} /tidb/cdc/default/__cdc_meta__/capture/fa7f932b-5d0c-4777-b1b4-e466e016f924 {"id":"fa7f932b-5d0c-4777-b1b4-e466e016f924","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885082} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d7cd fa7f932b-5d0c-4777-b1b4-e466e016f924 /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d801 8d7cfccf-1d50-4ef4-aedb-f735f7dbb4bf /tidb/cdc/default/__cdc_meta__/owner/22318f471d80d81c 9d40845a-d950-4202-9406-278a9b8b0f87 /tidb/cdc/default/default/upstream/7365375287312128763 {"id":7365375287312128763,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.canal_json_storage_partition_table.cli.10834.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml Create changefeed successfully! ID: 2dd55458-9773-4b00-be98-40c563d888e3 Info: {"upstream_id":7365375287312128763,"namespace":"default","id":"2dd55458-9773-4b00-be98-40c563d888e3","sink_uri":"file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:12.038912301+08:00","start_ts":449546837536800781,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546837536800781,"checkpoint_ts":449546837536800781,"checkpoint_time":"2024-05-05 12:58:11.922"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table multi_capture_4.usertable not exists for 4-th check, retry later wait process cdc.test exit for 1-th time... table batch_add_table.finish_mark not exists for 5-th check, retry later table capture_session_done_during_task.t exists check diff failed 1-th time, retry later wait process cdc.test exit for 2-th time... check diff failed 4-th time, retry later table owner_resign.t1 not exists for 5-th check, retry later + set +x table ddl_manager.finish_mark not exists for 43-th check, retry later + set +x cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:58:13 CST 2024] <<<<<< run test case tiflash success! >>>>>> table processor_resolved_ts_fallback.t1 not exists for 4-th check, retry later check diff failed 5-th time, retry later wait process cdc.test exit for 1-th time... ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 2-th time, retry later wait process cdc.test exit for 2-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_capture_4.usertable not exists for 5-th check, retry later table batch_add_table.finish_mark not exists for 6-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:58:14 CST 2024] <<<<<< run test case changefeed_error success! >>>>>> check diff failed 2-th time, retry later check diff failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:14 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2814e602-6467-4c76-b84a-7e1bcac211d1 {"id":"2814e602-6467-4c76-b84a-7e1bcac211d1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885092} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471da7afc6 2814e602-6467-4c76-b84a-7e1bcac211d1 /tidb/cdc/default/default/upstream/7365375333228691867 {"id":7365375333228691867,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2814e602-6467-4c76-b84a-7e1bcac211d1 {"id":"2814e602-6467-4c76-b84a-7e1bcac211d1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885092} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471da7afc6 2814e602-6467-4c76-b84a-7e1bcac211d1 /tidb/cdc/default/default/upstream/7365375333228691867 {"id":7365375333228691867,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/2814e602-6467-4c76-b84a-7e1bcac211d1 {"id":"2814e602-6467-4c76-b84a-7e1bcac211d1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885092} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471da7afc6 2814e602-6467-4c76-b84a-7e1bcac211d1 /tidb/cdc/default/default/upstream/7365375333228691867 {"id":7365375333228691867,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/processor_stop_delay + sink_uri='file:///tmp/tidb_cdc_test/processor_stop_delay/storage_test/ticdc-processor-stop-delay-31386?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_stop_delay ++ date + echo '[Sun May 5 12:58:14 CST 2024] <<<<<< START storage consumer in processor_stop_delay case >>>>>>' [Sun May 5 12:58:14 CST 2024] <<<<<< START storage consumer in processor_stop_delay case >>>>>> + cd /tmp/tidb_cdc_test/processor_stop_delay + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_stop_delay + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_stop_delay/storage_test/ticdc-processor-stop-delay-31386?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table owner_resign.t1 exists table processor_stop_delay.t not exists for 1-th check, retry later check diff failed 1-th time, retry later go: downloading github.com/googleapis/enterprise-certificate-proxy v0.3.2 go: downloading github.com/google/s2a-go v0.1.7 go: downloading go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 go: downloading go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 go: downloading github.com/felixge/httpsnoop v1.0.4 go: downloading github.com/jmespath/go-jmespath v0.4.0 table ddl_manager.finish_mark not exists for 44-th check, retry later table processor_resolved_ts_fallback.t1 not exists for 5-th check, retry later check diff failed 6-th time, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table multi_capture_4.usertable exists check diff successfully ***************** properties ***************** "scanproportion"="0" "updateproportion"="0" "mysql.user"="root" "dotransactions"="false" "workload"="core" "recordcount"="20" "insertproportion"="0" "requestdistribution"="uniform" "threadcount"="2" "readproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.db"="multi_capture_1" "readallfields"="true" "operationcount"="0" ********************************************** Run finished, takes 22.51243ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1432.3, Avg(us): 2198, Min(us): 594, Max(us): 8535, 95th(us): 9000, 99th(us): 9000 ***************** properties ***************** "updateproportion"="0" "recordcount"="20" "mysql.user"="root" "dotransactions"="false" "readproportion"="0" "threadcount"="2" "workload"="core" "mysql.db"="multi_capture_2" "scanproportion"="0" "operationcount"="0" "mysql.port"="4000" "mysql.host"="127.0.0.1" "requestdistribution"="uniform" "readallfields"="true" "insertproportion"="0" ********************************************** Run finished, takes 11.037494ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2062.8, Avg(us): 1030, Min(us): 497, Max(us): 1658, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "threadcount"="2" "dotransactions"="false" "requestdistribution"="uniform" "insertproportion"="0" "readproportion"="0" "operationcount"="0" "mysql.user"="root" "readallfields"="true" "updateproportion"="0" "scanproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "mysql.port"="4000" "recordcount"="20" "mysql.db"="multi_capture_3" ********************************************** Run finished, takes 10.937167ms INSERT - Takes(s): 0.0, Count: 20, OPS: 2050.6, Avg(us): 1038, Min(us): 624, Max(us): 1783, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "recordcount"="20" "updateproportion"="0" "operationcount"="0" "mysql.user"="root" "mysql.db"="multi_capture_4" "readproportion"="0" "workload"="core" "mysql.host"="127.0.0.1" "dotransactions"="false" "readallfields"="true" "requestdistribution"="uniform" "scanproportion"="0" "threadcount"="2" "mysql.port"="4000" "insertproportion"="0" ********************************************** Run finished, takes 11.428798ms INSERT - Takes(s): 0.0, Count: 20, OPS: 1986.3, Avg(us): 1045, Min(us): 435, Max(us): 2159, 95th(us): 2000, 99th(us): 3000 check diff failed 1-th time, retry later table batch_add_table.finish_mark exists check diff successfully table processor_stop_delay.t not exists for 2-th check, retry later wait process cdc.test exit for 1-th time... check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 45-th check, retry later check diff successfully ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 4-th time, retry later wait process cdc.test exit for 2-th time... check diff failed 2-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_resolved_ts_fallback.t1 exists [Sun May 5 12:58:17 CST 2024] <<<<<< START cdc server in processor_resolved_ts_fallback case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_resolved_ts_fallback.28712873.out server --log-file /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_resolved_ts_fallback/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:58:18 CST 2024] <<<<<< run test case batch_add_table success! >>>>>> check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 46-th check, retry later ERROR 1146 (42S02) at line 1: Table 'test.simple1' doesn't exist check data failed 5-th time, retry later table processor_stop_delay.t not exists for 3-th check, retry later check diff failed 3-th time, retry later check diff failed 7-th time, retry later check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later table ddl_manager.finish_mark not exists for 47-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:20 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/8bda9019-0498-47f2-8759-e93affd9c0c3 {UpstreamID:7365375277098550907 Namespace:default ID:8bda9019-0498-47f2-8759-e93affd9c0c3 SinkURI:file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:58:04.66179173 +0800 CST StartTs:449546835602178053 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014d4a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546835628654597} {CheckpointTs:449546839691886596 MinTableBarrierTs:449546839691886596 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/270e2320-4c51-40c0-92af-3e8ed63020e3 {"id":"270e2320-4c51-40c0-92af-3e8ed63020e3","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885097} /tidb/cdc/default/__cdc_meta__/capture/6b810204-8b8e-4ccf-91de-a7b26554b616 {"id":"6b810204-8b8e-4ccf-91de-a7b26554b616","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885081} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7cd2d1 6b810204-8b8e-4ccf-91de-a7b26554b616 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7cd398 270e2320-4c51-40c0-92af-3e8ed63020e3 /tidb/cdc/default/default/changefeed/info/8bda9019-0498-47f2-8759-e93affd9c0c3 {"upstream-id":7365375277098550907,"namespace":"default","changefeed-id":"8bda9019-0498-47f2-8759-e93affd9c0c3","sink-uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:58:04.66179173+08:00","start-ts":449546835602178053,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546835628654597} /tidb/cdc/default/default/changefeed/status/8bda9019-0498-47f2-8759-e93affd9c0c3 {"checkpoint-ts":449546839691886596,"min-table-barrier-ts":449546839691886596,"admin-job-type":0} /tidb/cdc/default/default/task/position/270e2320-4c51-40c0-92af-3e8ed63020e3/8bda9019-0498-47f2-8759-e93affd9c0c3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6b810204-8b8e-4ccf-91de-a7b26554b616/8bda9019-0498-47f2-8759-e93affd9c0c3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375277098550907 {"id":7365375277098550907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/8bda9019-0498-47f2-8759-e93affd9c0c3 {UpstreamID:7365375277098550907 Namespace:default ID:8bda9019-0498-47f2-8759-e93affd9c0c3 SinkURI:file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:58:04.66179173 +0800 CST StartTs:449546835602178053 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014d4a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546835628654597} {CheckpointTs:449546839691886596 MinTableBarrierTs:449546839691886596 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/270e2320-4c51-40c0-92af-3e8ed63020e3 {"id":"270e2320-4c51-40c0-92af-3e8ed63020e3","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885097} /tidb/cdc/default/__cdc_meta__/capture/6b810204-8b8e-4ccf-91de-a7b26554b616 {"id":"6b810204-8b8e-4ccf-91de-a7b26554b616","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885081} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7cd2d1 6b810204-8b8e-4ccf-91de-a7b26554b616 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7cd398 270e2320-4c51-40c0-92af-3e8ed63020e3 /tidb/cdc/default/default/changefeed/info/8bda9019-0498-47f2-8759-e93affd9c0c3 {"upstream-id":7365375277098550907,"namespace":"default","changefeed-id":"8bda9019-0498-47f2-8759-e93affd9c0c3","sink-uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:58:04.66179173+08:00","start-ts":449546835602178053,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546835628654597} /tidb/cdc/default/default/changefeed/status/8bda9019-0498-47f2-8759-e93affd9c0c3 {"checkpoint-ts":449546839691886596,"min-table-barrier-ts":449546839691886596,"admin-job-type":0} /tidb/cdc/default/default/task/position/270e2320-4c51-40c0-92af-3e8ed63020e3/8bda9019-0498-47f2-8759-e93affd9c0c3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6b810204-8b8e-4ccf-91de-a7b26554b616/8bda9019-0498-47f2-8759-e93affd9c0c3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375277098550907 {"id":7365375277098550907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/8bda9019-0498-47f2-8759-e93affd9c0c3 {UpstreamID:7365375277098550907 Namespace:default ID:8bda9019-0498-47f2-8759-e93affd9c0c3 SinkURI:file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:58:04.66179173 +0800 CST StartTs:449546835602178053 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc0014d4a20 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546835628654597} {CheckpointTs:449546839691886596 MinTableBarrierTs:449546839691886596 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/270e2320-4c51-40c0-92af-3e8ed63020e3 {"id":"270e2320-4c51-40c0-92af-3e8ed63020e3","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885097} /tidb/cdc/default/__cdc_meta__/capture/6b810204-8b8e-4ccf-91de-a7b26554b616 {"id":"6b810204-8b8e-4ccf-91de-a7b26554b616","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885081} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7cd2d1 6b810204-8b8e-4ccf-91de-a7b26554b616 /tidb/cdc/default/__cdc_meta__/owner/22318f471d7cd398 270e2320-4c51-40c0-92af-3e8ed63020e3 /tidb/cdc/default/default/changefeed/info/8bda9019-0498-47f2-8759-e93affd9c0c3 {"upstream-id":7365375277098550907,"namespace":"default","changefeed-id":"8bda9019-0498-47f2-8759-e93affd9c0c3","sink-uri":"file:///tmp/tidb_cdc_test/processor_resolved_ts_fallback/storage_test/ticdc-processor-resolved-ts-fallback-test-10949?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:58:04.66179173+08:00","start-ts":449546835602178053,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546835628654597} /tidb/cdc/default/default/changefeed/status/8bda9019-0498-47f2-8759-e93affd9c0c3 {"checkpoint-ts":449546839691886596,"min-table-barrier-ts":449546839691886596,"admin-job-type":0} /tidb/cdc/default/default/task/position/270e2320-4c51-40c0-92af-3e8ed63020e3/8bda9019-0498-47f2-8759-e93affd9c0c3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6b810204-8b8e-4ccf-91de-a7b26554b616/8bda9019-0498-47f2-8759-e93affd9c0c3 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375277098550907 {"id":7365375277098550907,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x ERROR 1146 (42S02) at line 1: Table 'test.simple2' doesn't exist check data failed 6-th time, retry later table processor_stop_delay.t not exists for 4-th check, retry later check diff failed 4-th time, retry later check diff failed 8-th time, retry later check diff failed 5-th time, retry later table processor_resolved_ts_fallback.t2 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c77fa2c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:10564, start at 2024-05-05 12:58:19.170134375 +0800 CST m=+5.230756752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:19.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:19.147 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:19.147 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c77fa2c0018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:10564, start at 2024-05-05 12:58:19.170134375 +0800 CST m=+5.230756752 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:19.177 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:19.147 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:19.147 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c77fba80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-n3b-lkvmc, pid:10641, start at 2024-05-05 12:58:19.263331137 +0800 CST m=+5.275119902 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:19.269 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:19.242 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:19.242 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/csv_storage_partition_table/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/csv_storage_partition_table/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 4-th time, retry later table processor_stop_delay.t not exists for 5-th check, retry later + workdir=/tmp/tidb_cdc_test/canal_json_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/canal_json_storage_partition_table ++ date + echo '[Sun May 5 12:58:22 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>>' [Sun May 5 12:58:22 CST 2024] <<<<<< START storage consumer in canal_json_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/canal_json_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/canal_json_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/canal_json_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/canal_json_storage_partition_table/conf/changefeed.toml check diff successfully table ddl_manager.finish_mark not exists for 48-th check, retry later table processor_resolved_ts_fallback.t2 not exists for 2-th check, retry later check data successfully check diff failed 5-th time, retry later check diff successfully check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... [Sun May 5 12:58:24 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.1203112033.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc0.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data0 --cluster-id default --addr 127.0.0.1:8300 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:58:25 CST 2024] <<<<<< run test case simple success! >>>>>> table processor_stop_delay.t exists check diff successfully check diff failed 1-th time, retry later check diff successfully table ddl_manager.finish_mark not exists for 49-th check, retry later wait process cdc.test exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/region_merge/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff successfully [Sun May 5 12:58:25 CST 2024] <<<<<< START cdc server in owner_resign case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerDontUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.owner_resign.31133115.out server --log-file /tmp/tidb_cdc_test/owner_resign/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/owner_resign/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later table processor_resolved_ts_fallback.t2 not exists for 3-th check, retry later wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_sequence/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:58:27 CST 2024] <<<<<< run test case multi_capture success! >>>>>> check diff failed 1-th time, retry later table ddl_manager.finish_mark not exists for 50-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:27 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50e137fa-7d30-4651-aa81-8e0d154a65b3 {"id":"50e137fa-7d30-4651-aa81-8e0d154a65b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885104} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd262cc 50e137fa-7d30-4651-aa81-8e0d154a65b3 /tidb/cdc/default/default/upstream/7365375379532126089 {"id":7365375379532126089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50e137fa-7d30-4651-aa81-8e0d154a65b3 {"id":"50e137fa-7d30-4651-aa81-8e0d154a65b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885104} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd262cc 50e137fa-7d30-4651-aa81-8e0d154a65b3 /tidb/cdc/default/default/upstream/7365375379532126089 {"id":7365375379532126089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/50e137fa-7d30-4651-aa81-8e0d154a65b3 {"id":"50e137fa-7d30-4651-aa81-8e0d154a65b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885104} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd262cc 50e137fa-7d30-4651-aa81-8e0d154a65b3 /tidb/cdc/default/default/upstream/7365375379532126089 {"id":7365375379532126089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun May 5 12:58:27 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.1208612088.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later check diff failed 3-th time, retry later table processor_resolved_ts_fallback.t2 exists table processor_resolved_ts_fallback.t3 exists cdc cli processor list --server http://127.0.0.1:8301 |jq '.|length'|grep -E '^2$' 2 run task successfully start tidb cluster in /tmp/tidb_cdc_test/region_merge Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 1-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 2-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 3-th time, retry later TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 4-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/3a476e71-db0b-4ab5-acd8-1cd706bf728c {UpstreamID:7365375270729748060 Namespace:default ID:3a476e71-db0b-4ab5-acd8-1cd706bf728c SinkURI:file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:58:03.239889058 +0800 CST StartTs:449546835239370755 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00149c000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546835252477958} {CheckpointTs:449546841832030214 MinTableBarrierTs:449546841832030214 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3b356f92-ef61-4409-955f-31db04476ce6 {"id":"3b356f92-ef61-4409-955f-31db04476ce6","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885106} /tidb/cdc/default/__cdc_meta__/capture/bc8eef2c-c83b-459b-ab50-c9a0dab11721 {"id":"bc8eef2c-c83b-459b-ab50-c9a0dab11721","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d779bce bc8eef2c-c83b-459b-ab50-c9a0dab11721 /tidb/cdc/default/__cdc_meta__/owner/22318f471d779d0d 3b356f92-ef61-4409-955f-31db04476ce6 /tidb/cdc/default/default/changefeed/info/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"upstream-id":7365375270729748060,"namespace":"default","changefeed-id":"3a476e71-db0b-4ab5-acd8-1cd706bf728c","sink-uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:58:03.239889058+08:00","start-ts":449546835239370755,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546835252477958} /tidb/cdc/default/default/changefeed/status/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"checkpoint-ts":449546841832030214,"min-table-barrier-ts":449546841832030214,"admin-job-type":0} /tidb/cdc/default/default/task/position/3b356f92-ef61-4409-955f-31db04476ce6/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bc8eef2c-c83b-459b-ab50-c9a0dab11721/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375270729748060 {"id":7365375270729748060,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/3a476e71-db0b-4ab5-acd8-1cd706bf728c {UpstreamID:7365375270729748060 Namespace:default ID:3a476e71-db0b-4ab5-acd8-1cd706bf728c SinkURI:file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:58:03.239889058 +0800 CST StartTs:449546835239370755 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00149c000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546835252477958} {CheckpointTs:449546841832030214 MinTableBarrierTs:449546841832030214 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3b356f92-ef61-4409-955f-31db04476ce6 {"id":"3b356f92-ef61-4409-955f-31db04476ce6","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885106} /tidb/cdc/default/__cdc_meta__/capture/bc8eef2c-c83b-459b-ab50-c9a0dab11721 {"id":"bc8eef2c-c83b-459b-ab50-c9a0dab11721","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d779bce bc8eef2c-c83b-459b-ab50-c9a0dab11721 /tidb/cdc/default/__cdc_meta__/owner/22318f471d779d0d 3b356f92-ef61-4409-955f-31db04476ce6 /tidb/cdc/default/default/changefeed/info/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"upstream-id":7365375270729748060,"namespace":"default","changefeed-id":"3a476e71-db0b-4ab5-acd8-1cd706bf728c","sink-uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:58:03.239889058+08:00","start-ts":449546835239370755,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546835252477958} /tidb/cdc/default/default/changefeed/status/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"checkpoint-ts":449546841832030214,"min-table-barrier-ts":449546841832030214,"admin-job-type":0} /tidb/cdc/default/default/task/position/3b356f92-ef61-4409-955f-31db04476ce6/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bc8eef2c-c83b-459b-ab50-c9a0dab11721/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375270729748060 {"id":7365375270729748060,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/3a476e71-db0b-4ab5-acd8-1cd706bf728c {UpstreamID:7365375270729748060 Namespace:default ID:3a476e71-db0b-4ab5-acd8-1cd706bf728c SinkURI:file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:58:03.239889058 +0800 CST StartTs:449546835239370755 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00149c000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546835252477958} {CheckpointTs:449546841832030214 MinTableBarrierTs:449546841832030214 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/3b356f92-ef61-4409-955f-31db04476ce6 {"id":"3b356f92-ef61-4409-955f-31db04476ce6","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885106} /tidb/cdc/default/__cdc_meta__/capture/bc8eef2c-c83b-459b-ab50-c9a0dab11721 {"id":"bc8eef2c-c83b-459b-ab50-c9a0dab11721","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885079} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471d779bce bc8eef2c-c83b-459b-ab50-c9a0dab11721 /tidb/cdc/default/__cdc_meta__/owner/22318f471d779d0d 3b356f92-ef61-4409-955f-31db04476ce6 /tidb/cdc/default/default/changefeed/info/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"upstream-id":7365375270729748060,"namespace":"default","changefeed-id":"3a476e71-db0b-4ab5-acd8-1cd706bf728c","sink-uri":"file:///tmp/tidb_cdc_test/owner_resign/storage_test/ticdc-owner-resign-test-18453?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:58:03.239889058+08:00","start-ts":449546835239370755,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546835252477958} /tidb/cdc/default/default/changefeed/status/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"checkpoint-ts":449546841832030214,"min-table-barrier-ts":449546841832030214,"admin-job-type":0} /tidb/cdc/default/default/task/position/3b356f92-ef61-4409-955f-31db04476ce6/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/bc8eef2c-c83b-459b-ab50-c9a0dab11721/3a476e71-db0b-4ab5-acd8-1cd706bf728c {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375270729748060 {"id":7365375270729748060,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$' 2 run task successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 table ddl_manager.finish_mark not exists for 51-th check, retry later killing cdc1: 2594 cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_sequence Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 2-th time, retry later check diff failed 3-th time, retry later check diff failed 4-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_rocks/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... TEST FAILED: OUTPUT DOES NOT CONTAIN 'id: 1' ____________________________________ ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ check data failed 5-th time, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:30 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13e394a1-4140-440e-afc1-c6f88e14ab72 {"id":"13e394a1-4140-440e-afc1-c6f88e14ab72","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885107} /tidb/cdc/default/__cdc_meta__/capture/50e137fa-7d30-4651-aa81-8e0d154a65b3 {"id":"50e137fa-7d30-4651-aa81-8e0d154a65b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885104} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd262cc 50e137fa-7d30-4651-aa81-8e0d154a65b3 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd26302 13e394a1-4140-440e-afc1-c6f88e14ab72 /tidb/cdc/default/default/upstream/7365375379532126089 {"id":7365375379532126089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13e394a1-4140-440e-afc1-c6f88e14ab72 {"id":"13e394a1-4140-440e-afc1-c6f88e14ab72","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885107} /tidb/cdc/default/__cdc_meta__/capture/50e137fa-7d30-4651-aa81-8e0d154a65b3 {"id":"50e137fa-7d30-4651-aa81-8e0d154a65b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885104} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd262cc 50e137fa-7d30-4651-aa81-8e0d154a65b3 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd26302 13e394a1-4140-440e-afc1-c6f88e14ab72 /tidb/cdc/default/default/upstream/7365375379532126089 {"id":7365375379532126089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13e394a1-4140-440e-afc1-c6f88e14ab72 {"id":"13e394a1-4140-440e-afc1-c6f88e14ab72","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885107} /tidb/cdc/default/__cdc_meta__/capture/50e137fa-7d30-4651-aa81-8e0d154a65b3 {"id":"50e137fa-7d30-4651-aa81-8e0d154a65b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885104} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd262cc 50e137fa-7d30-4651-aa81-8e0d154a65b3 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd26302 13e394a1-4140-440e-afc1-c6f88e14ab72 /tidb/cdc/default/default/upstream/7365375379532126089 {"id":7365375379532126089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun May 5 12:58:30 CST 2024] <<<<<< START cdc server in csv_storage_partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.1213212134.out server --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 52-th check, retry later check diff failed 3-th time, retry later table partition_table.t not exists for 1-th check, retry later check diff failed 4-th time, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 2-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully table partition_table.t not exists for 2-th check, retry later check diff failed 1-th time, retry later wait process cdc.test exit for 1-th time... start tidb cluster in /tmp/tidb_cdc_test/multi_rocks Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check data successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 4-th time, retry later wait process cdc.test exit for 2-th time... table ddl_manager.finish_mark not exists for 53-th check, retry later wait process cdc.test exit for 3-th time... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:33 GMT < Content-Length: 1756 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13e394a1-4140-440e-afc1-c6f88e14ab72 {"id":"13e394a1-4140-440e-afc1-c6f88e14ab72","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885107} /tidb/cdc/default/__cdc_meta__/capture/50e137fa-7d30-4651-aa81-8e0d154a65b3 {"id":"50e137fa-7d30-4651-aa81-8e0d154a65b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885104} /tidb/cdc/default/__cdc_meta__/capture/c23030f4-9b92-4ff7-8fae-0688dab05e95 {"id":"c23030f4-9b92-4ff7-8fae-0688dab05e95","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885112} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd262cc 50e137fa-7d30-4651-aa81-8e0d154a65b3 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd26302 13e394a1-4140-440e-afc1-c6f88e14ab72 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd2632b c23030f4-9b92-4ff7-8fae-0688dab05e95 /tidb/cdc/default/default/upstream/7365375379532126089 {"id":7365375379532126089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13e394a1-4140-440e-afc1-c6f88e14ab72 {"id":"13e394a1-4140-440e-afc1-c6f88e14ab72","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885107} /tidb/cdc/default/__cdc_meta__/capture/50e137fa-7d30-4651-aa81-8e0d154a65b3 {"id":"50e137fa-7d30-4651-aa81-8e0d154a65b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885104} /tidb/cdc/default/__cdc_meta__/capture/c23030f4-9b92-4ff7-8fae-0688dab05e95 {"id":"c23030f4-9b92-4ff7-8fae-0688dab05e95","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885112} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd262cc 50e137fa-7d30-4651-aa81-8e0d154a65b3 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd26302 13e394a1-4140-440e-afc1-c6f88e14ab72 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd2632b c23030f4-9b92-4ff7-8fae-0688dab05e95 /tidb/cdc/default/default/upstream/7365375379532126089 {"id":7365375379532126089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/13e394a1-4140-440e-afc1-c6f88e14ab72 {"id":"13e394a1-4140-440e-afc1-c6f88e14ab72","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885107} /tidb/cdc/default/__cdc_meta__/capture/50e137fa-7d30-4651-aa81-8e0d154a65b3 {"id":"50e137fa-7d30-4651-aa81-8e0d154a65b3","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885104} /tidb/cdc/default/__cdc_meta__/capture/c23030f4-9b92-4ff7-8fae-0688dab05e95 {"id":"c23030f4-9b92-4ff7-8fae-0688dab05e95","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885112} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd262cc 50e137fa-7d30-4651-aa81-8e0d154a65b3 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd26302 13e394a1-4140-440e-afc1-c6f88e14ab72 /tidb/cdc/default/__cdc_meta__/owner/22318f471dd2632b c23030f4-9b92-4ff7-8fae-0688dab05e95 /tidb/cdc/default/default/upstream/7365375379532126089 {"id":7365375379532126089,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.csv_storage_partition_table.cli.12203.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml Create changefeed successfully! ID: 28d61e23-0204-49a1-aeae-69d2785f9888 Info: {"upstream_id":7365375379532126089,"namespace":"default","id":"28d61e23-0204-49a1-aeae-69d2785f9888","sink_uri":"file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:33.856274431+08:00","start_ts":449546843271200769,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"csv","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":true,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546843271200769,"checkpoint_ts":449546843271200769,"checkpoint_time":"2024-05-05 12:58:33.797"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process cdc.test exit for 3-th time... check diff failed 5-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:58:34 CST 2024] <<<<<< run test case capture_session_done_during_task success! >>>>>> table partition_table.t exists table partition_table.t1 exists table partition_table.finish_mark not exists for 1-th check, retry later check diff failed 2-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:58:34 CST 2024] <<<<<< run test case ddl_puller_lag success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 5-th time, retry later table ddl_manager.finish_mark not exists for 54-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' Error: Get "http://127.0.0.1:8301/api/v2/captures": dial tcp 127.0.0.1:8301: connect: connection refused run task failed 3-th time, retry later check diff failed 6-th time, retry later table partition_table.finish_mark not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 3-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 55-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Debugger for raftstore-v2 is used Debugger for raftstore-v2 is used check diff failed 6-th time, retry later check diff failed 7-th time, retry later table partition_table.finish_mark not exists for 3-th check, retry later check diff failed 4-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status_with_redo + CDC_BINARY=cdc.test + SINK_TYPE=storage + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo The 1 times to try to start tidb cluster... table ddl_manager.finish_mark not exists for 56-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 7-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cdc_server_tips/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c792f2c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:7671, start at 2024-05-05 12:58:38.948187186 +0800 CST m=+5.428553992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:38.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:38.923 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:38.923 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 8-th time, retry later check diff failed 5-th time, retry later table partition_table.finish_mark not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7948c80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt, pid:3954, start at 2024-05-05 12:58:40.580432947 +0800 CST m=+5.411209402 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:40.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:40.562 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:40.562 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 8-th time, retry later table ddl_manager.finish_mark not exists for 57-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c792f2c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:7671, start at 2024-05-05 12:58:38.948187186 +0800 CST m=+5.428553992 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:38.954 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:38.923 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:38.923 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7930a00008 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:7748, start at 2024-05-05 12:58:39.024493895 +0800 CST m=+5.443151606 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:39.031 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:39.016 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:39.016 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_sequence/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_sequence/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff successfully table partition_table.finish_mark not exists for 5-th check, retry later cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^1$' 1 run task successfully check diff failed 1-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/cdc_server_tips Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff failed 6-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7948c80013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt, pid:3954, start at 2024-05-05 12:58:40.580432947 +0800 CST m=+5.411209402 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:40.586 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:40.562 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:40.562 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c794a280016 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-7fv-m7snt, pid:4042, start at 2024-05-05 12:58:40.682750038 +0800 CST m=+5.431674425 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:40.689 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:40.650 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:40.650 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/region_merge/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/region_merge/tiflash/log/error.log arg matches is ArgMatches { args: {"config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/region_merge/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 58-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9059.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 2-th time, retry later check diff failed 9-th time, retry later check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 6-th check, retry later check diff failed 7-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 59-th check, retry later + set +x + tso='449546845991206913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546845991206913 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 12:58:45 CST 2024] <<<<<< START cdc server in ddl_sequence case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.90969098.out server --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_sequence/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Sun May 5 12:58:45 CST 2024] <<<<<< START cdc server in region_merge case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.region_merge.52895291.out server --log-file /tmp/tidb_cdc_test/region_merge/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/region_merge/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 check diff failed 2-th time, retry later table partition_table.finish_mark not exists for 7-th check, retry later wait process cdc.test exit for 1-th time... check diff failed 3-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff successfully % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_only_block_related_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 60-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:58:47 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>> Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > check diff failed 3-th time, retry later < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:48 GMT < Content-Length: 861 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/123e4aa4-301c-4b92-8806-96f1e931095a {"id":"123e4aa4-301c-4b92-8806-96f1e931095a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885125} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e29223e 123e4aa4-301c-4b92-8806-96f1e931095a /tidb/cdc/default/default/upstream/7365375474083263529 {"id":7365375474083263529,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/123e4aa4-301c-4b92-8806-96f1e931095a {"id":"123e4aa4-301c-4b92-8806-96f1e931095a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885125} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e29223e 123e4aa4-301c-4b92-8806-96f1e931095a /tidb/cdc/default/default/upstream/7365375474083263529 {"id":7365375474083263529,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/123e4aa4-301c-4b92-8806-96f1e931095a {"id":"123e4aa4-301c-4b92-8806-96f1e931095a","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885125} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e29223e 123e4aa4-301c-4b92-8806-96f1e931095a /tidb/cdc/default/default/upstream/7365375474083263529 {"id":7365375474083263529,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x table partition_table.finish_mark not exists for 8-th check, retry later wait process cdc.test exit for 1-th time... Create changefeed successfully! ID: 11c6bb99-25c7-4c39-a33f-19af1dc8f820 Info: {"upstream_id":7365375474083263529,"namespace":"default","id":"11c6bb99-25c7-4c39-a33f-19af1dc8f820","sink_uri":"file:///tmp/tidb_cdc_test/region_merge/storage_test/ticdc-region-merge-test-11729?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:48.794414602+08:00","start_ts":449546847168233477,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546847168233477,"checkpoint_ts":449546847168233477,"checkpoint_time":"2024-05-05 12:58:48.663"} + workdir=/tmp/tidb_cdc_test/region_merge + sink_uri='file:///tmp/tidb_cdc_test/region_merge/storage_test/ticdc-region-merge-test-11729?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/region_merge ++ date + echo '[Sun May 5 12:58:48 CST 2024] <<<<<< START storage consumer in region_merge case >>>>>>' [Sun May 5 12:58:48 CST 2024] <<<<<< START storage consumer in region_merge case >>>>>> + cd /tmp/tidb_cdc_test/region_merge + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/region_merge + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/region_merge/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/region_merge/storage_test/ticdc-region-merge-test-11729?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/ddl_attributes/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 2-th time... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:58:48 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18303dff-91d0-4924-8f38-64a2473b38be {"id":"18303dff-91d0-4924-8f38-64a2473b38be","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885125} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e26aad5 18303dff-91d0-4924-8f38-64a2473b38be /tidb/cdc/default/default/upstream/7365375466349711966 {"id":7365375466349711966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18303dff-91d0-4924-8f38-64a2473b38be {"id":"18303dff-91d0-4924-8f38-64a2473b38be","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885125} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e26aad5 18303dff-91d0-4924-8f38-64a2473b38be /tidb/cdc/default/default/upstream/7365375466349711966 {"id":7365375466349711966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18303dff-91d0-4924-8f38-64a2473b38be {"id":"18303dff-91d0-4924-8f38-64a2473b38be","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885125} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e26aad5 18303dff-91d0-4924-8f38-64a2473b38be /tidb/cdc/default/default/upstream/7365375466349711966 {"id":7365375466349711966,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_sequence.cli.9156.out cli changefeed create --start-ts=449546845991206913 '--sink-uri=file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-12010?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: eb70a1ba-e5f1-4925-b9fa-5aaa38badaaf Info: {"upstream_id":7365375466349711966,"namespace":"default","id":"eb70a1ba-e5f1-4925-b9fa-5aaa38badaaf","sink_uri":"file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-12010?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:58:49.144384509+08:00","start_ts":449546845991206913,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546845991206913,"checkpoint_ts":449546845991206913,"checkpoint_time":"2024-05-05 12:58:44.173"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... split_and_random_merge scale: 20 table ddl_manager.finish_mark not exists for 61-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:58:49 CST 2024] <<<<<< run test case processor_resolved_ts_fallback success! >>>>>> start tidb cluster in /tmp/tidb_cdc_test/ddl_only_block_related_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 4-th time, retry later table partition_table.finish_mark not exists for 9-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x + workdir=/tmp/tidb_cdc_test/ddl_sequence + sink_uri='file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-12010?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_sequence ++ date + echo '[Sun May 5 12:58:50 CST 2024] <<<<<< START storage consumer in ddl_sequence case >>>>>>' [Sun May 5 12:58:50 CST 2024] <<<<<< START storage consumer in ddl_sequence case >>>>>> + cd /tmp/tidb_cdc_test/ddl_sequence + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_sequence + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_sequence/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_sequence/storage_test/ticdc-ddl-sequence-test-12010?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 62-th check, retry later check diff failed 5-th time, retry later table partition_table.finish_mark not exists for 10-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 1-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/ddl_attributes Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table ddl_manager.finish_mark not exists for 63-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a03c40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:4052, start at 2024-05-05 12:58:52.554910999 +0800 CST m=+5.186613459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:52.562 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:52.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:52.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a03c40018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:4052, start at 2024-05-05 12:58:52.554910999 +0800 CST m=+5.186613459 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:52.562 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:52.529 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:52.529 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a054c000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:4136, start at 2024-05-05 12:58:52.640100556 +0800 CST m=+5.219235715 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:52.646 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:52.627 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:52.627 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cdc_server_tips/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table partition_table.finish_mark not exists for 11-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Verifying downstream PD is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 2-th check, retry later check diff failed 6-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 64-th check, retry later *************************** 1. row *************************** count(distinct region_id): 4 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.cli.5389.out cli tso query --pd=http://127.0.0.1:2379 table partition_table.finish_mark not exists for 12-th check, retry later table ddl_sequence.finish_mark not exists for 3-th check, retry later check diff failed 7-th time, retry later check diff successfully Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a1d140011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:10252, start at 2024-05-05 12:58:54.170256685 +0800 CST m=+5.472393037 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:54.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:54.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:54.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a1d140011 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:10252, start at 2024-05-05 12:58:54.170256685 +0800 CST m=+5.472393037 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:54.180 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:54.149 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:54.149 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a1f680015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:10330, start at 2024-05-05 12:58:54.334950717 +0800 CST m=+5.585698554 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:54.342 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:54.348 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:54.348 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 65-th check, retry later + set +x + tso='449546849086078977 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546849086078977 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x try a VALID cdc server command [Sun May 5 12:58:57 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.54385440.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process cdc.test exit for 3-th time... table ddl_sequence.finish_mark not exists for 4-th check, retry later check diff failed 8-th time, retry later table partition_table.finish_mark not exists for 13-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:58:58 CST 2024] <<<<<< run test case owner_resign success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11698.out cli tso query --pd=http://127.0.0.1:2379 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 66-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_sequence.finish_mark not exists for 5-th check, retry later table partition_table.finish_mark not exists for 14-th check, retry later split_and_random_merge scale: 40 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a79bc001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44, pid:3901, start at 2024-05-05 12:59:00.126777359 +0800 CST m=+23.460738846 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:00.135 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:00.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:00.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a79bc001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44, pid:3901, start at 2024-05-05 12:59:00.126777359 +0800 CST m=+23.460738846 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:00.135 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:00.129 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:00.129 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c797a300005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-k2z-dxj44, pid:3978, start at 2024-05-05 12:58:43.727339047 +0800 CST m=+6.999924582 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:00:43.736 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:58:43.724 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:48:43.724 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/multi_rocks/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/db/proxy"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_rocks/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x + tso='449546849969766401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546849969766401 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449546849969766401 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sun May 5 12:59:00 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS= + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1173411736.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:59:00 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3bc0590-6bcf-4caa-9851-fb7cf09916fa {"id":"f3bc0590-6bcf-4caa-9851-fb7cf09916fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885137} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e58e3de f3bc0590-6bcf-4caa-9851-fb7cf09916fa /tidb/cdc/default/default/upstream/7365375526455887018 {"id":7365375526455887018,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3bc0590-6bcf-4caa-9851-fb7cf09916fa {"id":"f3bc0590-6bcf-4caa-9851-fb7cf09916fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885137} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e58e3de f3bc0590-6bcf-4caa-9851-fb7cf09916fa /tidb/cdc/default/default/upstream/7365375526455887018 {"id":7365375526455887018,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/f3bc0590-6bcf-4caa-9851-fb7cf09916fa {"id":"f3bc0590-6bcf-4caa-9851-fb7cf09916fa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885137} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e58e3de f3bc0590-6bcf-4caa-9851-fb7cf09916fa /tidb/cdc/default/default/upstream/7365375526455887018 {"id":7365375526455887018,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x check diff failed 9-th time, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 67-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 15-th check, retry later check diff successfully table ddl_sequence.finish_mark not exists for 6-th check, retry later [Sun May 5 12:59:03 CST 2024] <<<<<< START cdc server in multi_rocks case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.1666716669.out server --log-file /tmp/tidb_cdc_test/multi_rocks/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_rocks/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 68-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a8d9c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:6858, start at 2024-05-05 12:59:01.390249198 +0800 CST m=+5.543046638 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:01.399 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:01.401 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:01.401 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a8d9c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:6858, start at 2024-05-05 12:59:01.390249198 +0800 CST m=+5.543046638 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:01.399 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:01.401 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:01.401 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7a8f40000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:6939, start at 2024-05-05 12:59:01.466585254 +0800 CST m=+5.562780875 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:01.473 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:01.456 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:01.456 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_only_block_related_table/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:59:03 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18a52b1d-242a-4e62-8435-b7b0847b62e0 {"id":"18a52b1d-242a-4e62-8435-b7b0847b62e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e5c16d6 18a52b1d-242a-4e62-8435-b7b0847b62e0 /tidb/cdc/default/default/upstream/7365375528679261284 {"id":7365375528679261284,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18a52b1d-242a-4e62-8435-b7b0847b62e0 {"id":"18a52b1d-242a-4e62-8435-b7b0847b62e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e5c16d6 18a52b1d-242a-4e62-8435-b7b0847b62e0 /tidb/cdc/default/default/upstream/7365375528679261284 {"id":7365375528679261284,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/18a52b1d-242a-4e62-8435-b7b0847b62e0 {"id":"18a52b1d-242a-4e62-8435-b7b0847b62e0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885141} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e5c16d6 18a52b1d-242a-4e62-8435-b7b0847b62e0 /tidb/cdc/default/default/upstream/7365375528679261284 {"id":7365375528679261284,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449546849969766401 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.11792.out cli changefeed create --start-ts=449546849969766401 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365375528679261284,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T12:59:04.416458476+08:00","start_ts":449546849969766401,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546849969766401,"checkpoint_ts":449546849969766401,"checkpoint_time":"2024-05-05 12:58:59.350"} PASS table partition_table.finish_mark exists check diff successfully \033[0;36m<<< Run all test success >>>\033[0m coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } table ddl_sequence.finish_mark not exists for 7-th check, retry later [Pipeline] // withCredentials [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } table ddl_manager.finish_mark not exists for 69-th check, retry later [Pipeline] // node [Pipeline] } check diff failed 1-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/partition_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... wait process cdc.test exit for 2-th time... [Pipeline] // podTemplate VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7abf0c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51, pid:4317, start at 2024-05-05 12:59:04.536430516 +0800 CST m=+5.341714260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:04.543 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:04.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:04.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7abf0c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51, pid:4317, start at 2024-05-05 12:59:04.536430516 +0800 CST m=+5.341714260 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:04.543 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:04.515 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:04.515 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7abfd00009 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-sx9-8fp51, pid:4409, start at 2024-05-05 12:59:04.574272625 +0800 CST m=+5.326679849 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:04.580 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:04.564 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:04.564 +0800 All versions after safe point can be accessed. (DO NOT EDIT) [Pipeline] } [Pipeline] // withEnv [Pipeline] } Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/ddl_attributes/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_attributes/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Pipeline] // stage [Pipeline] } + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2326 0 --:--:-- --:--:-- --:--:-- 2351 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-05 12:58:59.350","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-05 12:59:06.000","info":"Data syncing is finished"}' ++ jq .synced ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '12:58:59.350","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '12:59:06.000","info":"Data' syncing is 'finished"}' wait process cdc.test exit for 3-th time... < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:59:06 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc8c8bd9-d3e0-41c9-8263-3e7dea5756d5 {"id":"cc8c8bd9-d3e0-41c9-8263-3e7dea5756d5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885143} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e32fdf4 cc8c8bd9-d3e0-41c9-8263-3e7dea5756d5 /tidb/cdc/default/default/upstream/7365375478590573633 {"id":7365375478590573633,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc8c8bd9-d3e0-41c9-8263-3e7dea5756d5 {"id":"cc8c8bd9-d3e0-41c9-8263-3e7dea5756d5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885143} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e32fdf4 cc8c8bd9-d3e0-41c9-8263-3e7dea5756d5 /tidb/cdc/default/default/upstream/7365375478590573633 {"id":7365375478590573633,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cc8c8bd9-d3e0-41c9-8263-3e7dea5756d5 {"id":"cc8c8bd9-d3e0-41c9-8263-3e7dea5756d5","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885143} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e32fdf4 cc8c8bd9-d3e0-41c9-8263-3e7dea5756d5 /tidb/cdc/default/default/upstream/7365375478590573633 {"id":7365375478590573633,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_rocks.cli.16945.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-10205?protocol=canal-json&enable-tidb-extension=true' + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '12:58:59.350","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '12:59:06.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-05 12:58:59.350' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '12:58:59.350","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '12:59:06.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '12:58:59.350","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '12:59:06.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-05 12:59:06' + echo 'sink_checkpoint_ts is 2024-05-05' 12:58:59.350 sink_checkpoint_ts is 2024-05-05 12:58:59.350 ++ date -d '2024-05-05 12:58:59.350' +%s + checkpoint_timestamp=1714885139 ++ date -d '2024-05-05 12:59:06' +%s + current_timestamp=1714885146 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' start tidb cluster in /tmp/tidb_cdc_test/partition_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Create changefeed successfully! ID: f8b2b5dd-fed7-48d5-ab38-613ad02dc969 Info: {"upstream_id":7365375478590573633,"namespace":"default","id":"f8b2b5dd-fed7-48d5-ab38-613ad02dc969","sink_uri":"file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-10205?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:59:06.528950751+08:00","start_ts":449546851825483782,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546851825483782,"checkpoint_ts":449546851825483782,"checkpoint_time":"2024-05-05 12:59:06.429"} PASS [Sun May 5 12:59:06 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.81958197.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:59:06 CST 2024] <<<<<< run test case canal_json_storage_partition_table success! >>>>>> coverage: 2.5% of statements in github.com/pingcap/tiflow/... table ddl_sequence.finish_mark not exists for 8-th check, retry later + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + workdir=/tmp/tidb_cdc_test/csv_storage_partition_table + sink_uri='file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/csv_storage_partition_table ++ date + echo '[Sun May 5 12:59:06 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>>' [Sun May 5 12:59:06 CST 2024] <<<<<< START storage consumer in csv_storage_partition_table case >>>>>> + cd /tmp/tidb_cdc_test/csv_storage_partition_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/csv_storage_partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/csv_storage_partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/csv_storage_partition_table/storage_test?flush-interval=5s&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/csv_storage_partition_table/conf/changefeed.toml check diff failed 2-th time, retry later table ddl_manager.finish_mark not exists for 70-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/multi_rocks + sink_uri='file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-10205?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/multi_rocks ++ date + echo '[Sun May 5 12:59:07 CST 2024] <<<<<< START storage consumer in multi_rocks case >>>>>>' [Sun May 5 12:59:07 CST 2024] <<<<<< START storage consumer in multi_rocks case >>>>>> + cd /tmp/tidb_cdc_test/multi_rocks + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/multi_rocks + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/multi_rocks/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/multi_rocks/storage_test/ticdc-multi-rocks-test-10205?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.5721.out cli tso query --pd=http://127.0.0.1:2379 table test.t1 exists + sleep 5 table ddl_sequence.finish_mark not exists for 9-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release ***************** properties ***************** "scanproportion"="0" "operationcount"="0" "dotransactions"="false" "threadcount"="2" "readallfields"="true" "mysql.host"="127.0.0.1" "mysql.user"="root" "mysql.db"="multi_rocks" "mysql.port"="4000" "workload"="core" "updateproportion"="0" "insertproportion"="0" "recordcount"="1000" "table"="a1" "readproportion"="0" "requestdistribution"="uniform" ********************************************** check diff failed 3-th time, retry later Run finished, takes 531.953788ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1922.0, Avg(us): 1029, Min(us): 783, Max(us): 12661, 95th(us): 2000, 99th(us): 2000 + set +x + tso='449546852280565762 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546852280565762 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 12:59:09 CST 2024] <<<<<< START cdc server in ddl_attributes case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.57635765.out server --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_attributes/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 71-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:59:09 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2c61fe0-0326-4c24-bfbb-e08317b627b4 {"id":"d2c61fe0-0326-4c24-bfbb-e08317b627b4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885146} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e7611cc d2c61fe0-0326-4c24-bfbb-e08317b627b4 /tidb/cdc/default/default/upstream/7365375559109997918 {"id":7365375559109997918,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2c61fe0-0326-4c24-bfbb-e08317b627b4 {"id":"d2c61fe0-0326-4c24-bfbb-e08317b627b4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885146} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e7611cc d2c61fe0-0326-4c24-bfbb-e08317b627b4 /tidb/cdc/default/default/upstream/7365375559109997918 {"id":7365375559109997918,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2c61fe0-0326-4c24-bfbb-e08317b627b4 {"id":"d2c61fe0-0326-4c24-bfbb-e08317b627b4","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885146} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e7611cc d2c61fe0-0326-4c24-bfbb-e08317b627b4 /tidb/cdc/default/default/upstream/7365375559109997918 {"id":7365375559109997918,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.cli.8264.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json&enable-tidb-extension=true' -c=ddl-only-block-related-table Create changefeed successfully! ID: ddl-only-block-related-table Info: {"upstream_id":7365375559109997918,"namespace":"default","id":"ddl-only-block-related-table","sink_uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:59:09.698740728+08:00","start_ts":449546852657266693,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546852657266693,"checkpoint_ts":449546852657266693,"checkpoint_time":"2024-05-05 12:59:09.602"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ***************** properties ***************** "workload"="core" "recordcount"="1000" "mysql.port"="4000" "dotransactions"="false" "requestdistribution"="uniform" "table"="a2" "readproportion"="0" "mysql.user"="root" "operationcount"="0" "threadcount"="2" "mysql.host"="127.0.0.1" "insertproportion"="0" "scanproportion"="0" "readallfields"="true" "updateproportion"="0" "mysql.db"="multi_rocks" ********************************************** table ddl_sequence.finish_mark not exists for 10-th check, retry later Run finished, takes 555.405589ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1856.3, Avg(us): 1080, Min(us): 791, Max(us): 16623, 95th(us): 2000, 99th(us): 2000 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ***************** properties ***************** "updateproportion"="0" "table"="a3" "recordcount"="1000" "scanproportion"="0" "mysql.host"="127.0.0.1" "mysql.user"="root" "mysql.port"="4000" "insertproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "workload"="core" "mysql.db"="multi_rocks" "dotransactions"="false" "readallfields"="true" "operationcount"="0" "readproportion"="0" ********************************************** + set +x + workdir=/tmp/tidb_cdc_test/ddl_only_block_related_table + sink_uri='file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_only_block_related_table ++ date + echo '[Sun May 5 12:59:11 CST 2024] <<<<<< START storage consumer in ddl_only_block_related_table case >>>>>>' [Sun May 5 12:59:11 CST 2024] <<<<<< START storage consumer in ddl_only_block_related_table case >>>>>> + cd /tmp/tidb_cdc_test/ddl_only_block_related_table + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_only_block_related_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 72-th check, retry later check diff failed 4-th time, retry later Run finished, takes 546.426052ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 1912.7, Avg(us): 1062, Min(us): 778, Max(us): 23491, 95th(us): 2000, 99th(us): 2000 ***************** properties ***************** "scanproportion"="0" "table"="a4" "recordcount"="1000" "updateproportion"="0" "threadcount"="2" "requestdistribution"="uniform" "mysql.port"="4000" "mysql.user"="root" "insertproportion"="0" "readallfields"="true" "mysql.db"="multi_rocks" "operationcount"="0" "mysql.host"="127.0.0.1" "readproportion"="0" "dotransactions"="false" "workload"="core" ********************************************** Run finished, takes 470.378225ms INSERT - Takes(s): 0.5, Count: 999, OPS: 2201.4, Avg(us): 897, Min(us): 690, Max(us): 16460, 95th(us): 2000, 99th(us): 2000 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:59:12 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f7dc503-82b8-4c75-a533-565be054b9ca {"id":"5f7dc503-82b8-4c75-a533-565be054b9ca","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e8306d3 5f7dc503-82b8-4c75-a533-565be054b9ca /tidb/cdc/default/default/upstream/7365375566741512628 {"id":7365375566741512628,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f7dc503-82b8-4c75-a533-565be054b9ca {"id":"5f7dc503-82b8-4c75-a533-565be054b9ca","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e8306d3 5f7dc503-82b8-4c75-a533-565be054b9ca /tidb/cdc/default/default/upstream/7365375566741512628 {"id":7365375566741512628,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/5f7dc503-82b8-4c75-a533-565be054b9ca {"id":"5f7dc503-82b8-4c75-a533-565be054b9ca","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885149} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e8306d3 5f7dc503-82b8-4c75-a533-565be054b9ca /tidb/cdc/default/default/upstream/7365375566741512628 {"id":7365375566741512628,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_attributes.cli.5823.out cli changefeed create --start-ts=449546852280565762 '--sink-uri=file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-8230?protocol=canal-json&enable-tidb-extension=true' table ddl_sequence.finish_mark not exists for 11-th check, retry later Create changefeed successfully! ID: aef08017-6224-486a-87b7-b11e3ef4fd98 Info: {"upstream_id":7365375566741512628,"namespace":"default","id":"aef08017-6224-486a-87b7-b11e3ef4fd98","sink_uri":"file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-8230?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:59:13.145796881+08:00","start_ts":449546852280565762,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546852280565762,"checkpoint_ts":449546852280565762,"checkpoint_time":"2024-05-05 12:59:08.165"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ***************** properties ***************** "dotransactions"="false" "workload"="core" "scanproportion"="0" "insertproportion"="0" "updateproportion"="0" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.user"="root" "mysql.db"="multi_rocks" "readallfields"="true" "table"="a5" "operationcount"="0" "requestdistribution"="uniform" "threadcount"="2" "recordcount"="1000" "readproportion"="0" ********************************************** table ddl_manager.finish_mark not exists for 73-th check, retry later check diff failed 5-th time, retry later table ddl_only_block_related_table.finish_mark not exists for 1-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2385 0 --:--:-- --:--:-- --:--:-- 2405 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 12:59:11.750","puller_resolved_ts":"2024-05-05 12:59:06.200","last_synced_ts":"2024-05-05 12:59:06.700","now_ts":"2024-05-05 12:59:13.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '12:59:11.750","puller_resolved_ts":"2024-05-05' '12:59:06.200","last_synced_ts":"2024-05-05' '12:59:06.700","now_ts":"2024-05-05' '12:59:13.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '12:59:11.750","puller_resolved_ts":"2024-05-05' '12:59:06.200","last_synced_ts":"2024-05-05' '12:59:06.700","now_ts":"2024-05-05' '12:59:13.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 Run finished, takes 515.491319ms INSERT - Takes(s): 0.5, Count: 1000, OPS: 2001.0, Avg(us): 1002, Min(us): 735, Max(us): 15649, 95th(us): 2000, 99th(us): 2000 + set +x + workdir=/tmp/tidb_cdc_test/ddl_attributes + sink_uri='file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-8230?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/ddl_attributes ++ date + echo '[Sun May 5 12:59:14 CST 2024] <<<<<< START storage consumer in ddl_attributes case >>>>>>' [Sun May 5 12:59:14 CST 2024] <<<<<< START storage consumer in ddl_attributes case >>>>>> + cd /tmp/tidb_cdc_test/ddl_attributes + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/ddl_attributes + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/ddl_attributes/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/ddl_attributes/storage_test/ticdc-ddl-attributes-test-8230?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table ddl_sequence.finish_mark not exists for 12-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 74-th check, retry later table multi_rocks.finish_mark not exists for 1-th check, retry later check diff failed 6-th time, retry later table ddl_only_block_related_table.finish_mark not exists for 2-th check, retry later table ddl_sequence.finish_mark not exists for 13-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7b74b80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:9583, start at 2024-05-05 12:59:16.143899291 +0800 CST m=+5.158454871 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:16.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:16.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:16.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7b74b80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:9583, start at 2024-05-05 12:59:16.143899291 +0800 CST m=+5.158454871 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:16.151 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:16.142 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:16.142 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7b73c40017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-r7g-5j9d0, pid:9654, start at 2024-05-05 12:59:16.128870513 +0800 CST m=+5.089134472 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:16.135 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:16.132 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:16.132 +0800 All versions after safe point can be accessed. (DO NOT EDIT) table partition_table.t not exists for 2-th check, retry later Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/partition_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/partition_table/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/partition_table/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table ddl_manager.finish_mark not exists for 75-th check, retry later table multi_rocks.finish_mark not exists for 2-th check, retry later check diff failed 7-th time, retry later table ddl_only_block_related_table.finish_mark not exists for 3-th check, retry later table ddl_sequence.finish_mark not exists for 14-th check, retry later table partition_table.t exists table partition_table.t1 exists table partition_table.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 76-th check, retry later table multi_rocks.finish_mark not exists for 3-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11022.out cli tso query --pd=http://127.0.0.1:2379 check diff failed 8-th time, retry later table ddl_only_block_related_table.finish_mark not exists for 4-th check, retry later table ddl_sequence.finish_mark not exists for 15-th check, retry later table partition_table.finish_mark not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + set +x + tso='449546855276085249 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546855276085249 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 12:59:21 CST 2024] <<<<<< START cdc server in partition_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.1106111063.out server --log-file /tmp/tidb_cdc_test/partition_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/partition_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 77-th check, retry later table multi_rocks.finish_mark not exists for 4-th check, retry later check diff failed 9-th time, retry later Verifying downstream PD is started... table ddl_only_block_related_table.finish_mark not exists for 5-th check, retry later table ddl_attributes.attributes_t1_new not exists for 1-th check, retry later table partition_table.finish_mark not exists for 3-th check, retry later valid ~~~ running cdc Failed to start cdc, the usage tips should be printed 1st test case cdc_server_tips success! try an INVALID cdc server command [Sun May 5 12:59:20 CST 2024] <<<<<< START cdc server in cdc_server_tips case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ true != \n\o ]] + set +x + GO_FAILPOINTS= + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cdc_server_tips.55235525.out server --log-file /tmp/tidb_cdc_test/cdc_server_tips/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cdc_server_tips/cdc_data --cluster-id default --pd None table ddl_sequence.finish_mark not exists for 16-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table ddl_manager.finish_mark not exists for 78-th check, retry later table multi_rocks.finish_mark not exists for 5-th check, retry later table ddl_only_block_related_table.finish_mark exists table ddl_attributes.attributes_t1_new not exists for 2-th check, retry later check diff successfully + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:59:24 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40404e3c-5d4d-4c79-844b-5f7e5a992db1 {"id":"40404e3c-5d4d-4c79-844b-5f7e5a992db1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885161} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471eb712ce 40404e3c-5d4d-4c79-844b-5f7e5a992db1 /tidb/cdc/default/default/upstream/7365375624342983025 {"id":7365375624342983025,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40404e3c-5d4d-4c79-844b-5f7e5a992db1 {"id":"40404e3c-5d4d-4c79-844b-5f7e5a992db1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885161} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471eb712ce 40404e3c-5d4d-4c79-844b-5f7e5a992db1 /tidb/cdc/default/default/upstream/7365375624342983025 {"id":7365375624342983025,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/40404e3c-5d4d-4c79-844b-5f7e5a992db1 {"id":"40404e3c-5d4d-4c79-844b-5f7e5a992db1","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885161} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471eb712ce 40404e3c-5d4d-4c79-844b-5f7e5a992db1 /tidb/cdc/default/default/upstream/7365375624342983025 {"id":7365375624342983025,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.partition_table.cli.11120.out cli changefeed create --start-ts=449546855276085249 '--sink-uri=file:///tmp/tidb_cdc_test/partition_table/storage_test/ticdc-partition-table-test-20711?protocol=canal-json&enable-tidb-extension=true' wait process 8200 exit for 1-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_tables_ddl/run.sh using Sink-Type: storage... <<================= * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:24927; Connection refused * Closing connection 0 You are running an older version of MinIO released 3 years ago Update: Run `mc admin update` Attempting encryption of all config, IAM users and policies on MinIO backend Endpoint: http://127.0.0.1:24927 Object API (Amazon S3 compatible): Go: https://docs.min.io/docs/golang-client-quickstart-guide Java: https://docs.min.io/docs/java-client-quickstart-guide Python: https://docs.min.io/docs/python-client-quickstart-guide JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide .NET: https://docs.min.io/docs/dotnet-client-quickstart-guide * About to connect() to 127.0.0.1 port 24927 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0) > GET / HTTP/1.1 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:24927 > Accept: */* > < HTTP/1.1 403 Forbidden < Accept-Ranges: bytes < Content-Length: 226 < Content-Security-Policy: block-all-mixed-content < Content-Type: application/xml < Server: MinIO/RELEASE.2020-07-27T18-37-02Z < Vary: Origin < X-Amz-Request-Id: 17CC7EF88B91FC33 < X-Xss-Protection: 1; mode=block < Date: Sun, 05 May 2024 04:59:24 GMT < { [data not shown] * Connection #0 to host 127.0.0.1 left intact Bucket 's3://logbucket/' created [Sun May 5 12:59:24 CST 2024] <<<<<< run test case multi_tables_ddl success! >>>>>> Exiting on signal: INTERRUPT Create changefeed successfully! ID: fb6e54b4-cc81-499a-a261-7d10f233ebc9 Info: {"upstream_id":7365375624342983025,"namespace":"default","id":"fb6e54b4-cc81-499a-a261-7d10f233ebc9","sink_uri":"file:///tmp/tidb_cdc_test/partition_table/storage_test/ticdc-partition-table-test-20711?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T12:59:24.585358118+08:00","start_ts":449546855276085249,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546855276085249,"checkpoint_ts":449546855276085249,"checkpoint_time":"2024-05-05 12:59:19.592"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... wait process 8200 exit for 2-th time... table ddl_sequence.finish_mark not exists for 17-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) wait process 8200 exit for 3-th time... table partition_table.finish_mark not exists for 4-th check, retry later table multi_rocks.finish_mark not exists for 6-th check, retry later check diff failed 1-th time, retry later table ddl_attributes.attributes_t1_new not exists for 3-th check, retry later /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8200) - No such process wait process 8200 exit for 4-th time... process 8200 already exit [Sun May 5 12:59:25 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ExecuteNotDone=return(true)' + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.84008402.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table ddl_manager.finish_mark not exists for 79-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/partition_table + sink_uri='file:///tmp/tidb_cdc_test/partition_table/storage_test/ticdc-partition-table-test-20711?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/partition_table ++ date + echo '[Sun May 5 12:59:26 CST 2024] <<<<<< START storage consumer in partition_table case >>>>>>' [Sun May 5 12:59:26 CST 2024] <<<<<< START storage consumer in partition_table case >>>>>> + cd /tmp/tidb_cdc_test/partition_table + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/partition_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/partition_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/partition_table/storage_test/ticdc-partition-table-test-20711?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table ddl_sequence.finish_mark not exists for 18-th check, retry later table partition_table.finish_mark not exists for 5-th check, retry later check diff failed 2-th time, retry later table ddl_attributes.attributes_t1_new not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 80-th check, retry later table multi_rocks.finish_mark not exists for 7-th check, retry later table ddl_sequence.finish_mark not exists for 19-th check, retry later table partition_table.finish_mark not exists for 6-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:59:28 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365375559109997918 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:59:09.698740728 +0800 CST StartTs:449546852657266693 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003da7560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546852670111752} {CheckpointTs:449546856143519748 MinTableBarrierTs:449546857454239748 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83f58769-9514-414a-bf4a-278e0445c664 {"id":"83f58769-9514-414a-bf4a-278e0445c664","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885166} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e7612c8 83f58769-9514-414a-bf4a-278e0445c664 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365375559109997918,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:59:09.698740728+08:00","start-ts":449546852657266693,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546852670111752} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449546856143519748,"min-table-barrier-ts":449546857454239748,"admin-job-type":0} /tidb/cdc/default/default/task/position/83f58769-9514-414a-bf4a-278e0445c664/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375559109997918 {"id":7365375559109997918,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365375559109997918 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:59:09.698740728 +0800 CST StartTs:449546852657266693 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003da7560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546852670111752} {CheckpointTs:449546856143519748 MinTableBarrierTs:449546857454239748 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83f58769-9514-414a-bf4a-278e0445c664 {"id":"83f58769-9514-414a-bf4a-278e0445c664","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885166} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e7612c8 83f58769-9514-414a-bf4a-278e0445c664 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365375559109997918,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:59:09.698740728+08:00","start-ts":449546852657266693,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546852670111752} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449546856143519748,"min-table-barrier-ts":449546857454239748,"admin-job-type":0} /tidb/cdc/default/default/task/position/83f58769-9514-414a-bf4a-278e0445c664/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375559109997918 {"id":7365375559109997918,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365375559109997918 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:59:09.698740728 +0800 CST StartTs:449546852657266693 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003da7560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546852670111752} {CheckpointTs:449546856143519748 MinTableBarrierTs:449546857454239748 AdminJobType:noop} span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449546856143519748, checkpointTs: 449546856143519748, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/83f58769-9514-414a-bf4a-278e0445c664 {"id":"83f58769-9514-414a-bf4a-278e0445c664","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885166} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e7612c8 83f58769-9514-414a-bf4a-278e0445c664 /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365375559109997918,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:59:09.698740728+08:00","start-ts":449546852657266693,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546852670111752} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table + grep -q 'etcd info' {"checkpoint-ts":449546856143519748,"min-table-barrier-ts":449546857454239748,"admin-job-type":0} /tidb/cdc/default/default/task/position/83f58769-9514-414a-bf4a-278e0445c664/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375559109997918 {"id":7365375559109997918,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check_ts_not_forward ddl-only-block-related-table ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 81-th check, retry later table multi_rocks.finish_mark not exists for 8-th check, retry later check diff failed 3-th time, retry later table ddl_attributes.attributes_t1_new not exists for 5-th check, retry later table ddl_sequence.finish_mark not exists for 20-th check, retry later run task failed 1-th time, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7c571c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc, pid:4507, start at 2024-05-05 12:59:30.664498946 +0800 CST m=+5.435061725 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:30.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:30.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:30.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table ddl_manager.finish_mark not exists for 82-th check, retry later table multi_rocks.finish_mark not exists for 9-th check, retry later check diff failed 4-th time, retry later table ddl_attributes.attributes_t1_new not exists for 6-th check, retry later table partition_table.finish_mark not exists for 7-th check, retry later table ddl_sequence.finish_mark not exists for 21-th check, retry later check_ts_not_forward ddl-only-block-related-table VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7c571c0012 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc, pid:4507, start at 2024-05-05 12:59:30.664498946 +0800 CST m=+5.435061725 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:30.671 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:30.631 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:30.631 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7c59c00014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc, pid:4585, start at 2024-05-05 12:59:30.825632888 +0800 CST m=+5.540241074 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:01:30.832 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-12:59:30.800 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:49:30.800 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... table ddl_manager.finish_mark not exists for 83-th check, retry later table multi_rocks.finish_mark not exists for 10-th check, retry later TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } check diff failed 5-th time, retry later table ddl_attributes.attributes_t1_new not exists for 7-th check, retry later table partition_table.finish_mark not exists for 8-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_sequence.finish_mark not exists for 22-th check, retry later [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } table multi_rocks.finish_mark not exists for 11-th check, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Sun May 5 12:59:35 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.58925894.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + [[ no != \n\o ]] + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [Pipeline] // stage [Pipeline] } table ddl_attributes.attributes_t1_new exists table ddl_attributes.finish_mark not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 84-th check, retry later table partition_table.finish_mark not exists for 9-th check, retry later check diff failed 6-th time, retry later table ddl_sequence.finish_mark not exists for 23-th check, retry later table ddl_manager.finish_mark not exists for 85-th check, retry later table multi_rocks.finish_mark not exists for 12-th check, retry later table partition_table.finish_mark not exists for 10-th check, retry later check diff failed 7-th time, retry later table ddl_attributes.finish_mark not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 04:59:38 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54fde1ba-e1be-486f-b90f-65bc0804a59d {"id":"54fde1ba-e1be-486f-b90f-65bc0804a59d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885175} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471eea97ce 54fde1ba-e1be-486f-b90f-65bc0804a59d /tidb/cdc/default/default/upstream/7365375686631296560 {"id":7365375686631296560,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54fde1ba-e1be-486f-b90f-65bc0804a59d {"id":"54fde1ba-e1be-486f-b90f-65bc0804a59d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885175} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471eea97ce 54fde1ba-e1be-486f-b90f-65bc0804a59d /tidb/cdc/default/default/upstream/7365375686631296560 {"id":7365375686631296560,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/54fde1ba-e1be-486f-b90f-65bc0804a59d {"id":"54fde1ba-e1be-486f-b90f-65bc0804a59d","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885175} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471eea97ce 54fde1ba-e1be-486f-b90f-65bc0804a59d /tidb/cdc/default/default/upstream/7365375686631296560 {"id":7365375686631296560,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/processor_etcd_worker_delay + sink_uri='file:///tmp/tidb_cdc_test/processor_etcd_worker_delay/storage_test/processor-etcd-delay-2188?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/processor_etcd_worker_delay ++ date + echo '[Sun May 5 12:59:38 CST 2024] <<<<<< START storage consumer in processor_etcd_worker_delay case >>>>>>' [Sun May 5 12:59:38 CST 2024] <<<<<< START storage consumer in processor_etcd_worker_delay case >>>>>> + cd /tmp/tidb_cdc_test/processor_etcd_worker_delay + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/processor_etcd_worker_delay + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/processor_etcd_worker_delay/storage_test/processor-etcd-delay-2188?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table ddl_sequence.finish_mark not exists for 24-th check, retry later table ddl_manager.finish_mark not exists for 86-th check, retry later table multi_rocks.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... check diff failed 8-th time, retry later table ddl_attributes.finish_mark not exists for 3-th check, retry later table partition_table.finish_mark not exists for 11-th check, retry later wait process cdc.test exit for 2-th time... table ddl_sequence.finish_mark not exists for 25-th check, retry later invalid ~~~ running cdc Failed to start cdc, the usage tips should be printed 2nd test case cdc_server_tips success! [Sun May 5 12:59:40 CST 2024] <<<<<< run all test cases cdc_server_tips success! >>>>>> cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:59:41 CST 2024] <<<<<< run test case multi_rocks success! >>>>>> table ddl_manager.finish_mark not exists for 87-th check, retry later table ddl_attributes.finish_mark not exists for 4-th check, retry later table partition_table.finish_mark not exists for 12-th check, retry later table ddl_sequence.finish_mark exists check diff successfully check diff successfully wait process cdc.test exit for 1-th time... table ddl_manager.finish_mark not exists for 88-th check, retry later wait process cdc.test exit for 2-th time... table partition_table.t exists table partition_table.t1 exists table partition_table.t2 not exists for 1-th check, retry later table ddl_attributes.finish_mark not exists for 5-th check, retry later check diff failed 1-th time, retry later table partition_table.finish_mark not exists for 13-th check, retry later wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:59:44 CST 2024] <<<<<< run test case ddl_sequence success! >>>>>> table partition_table.t2 not exists for 2-th check, retry later table ddl_attributes.finish_mark not exists for 6-th check, retry later table ddl_manager.finish_mark not exists for 89-th check, retry later table partition_table.finish_mark not exists for 14-th check, retry later check diff failed 2-th time, retry later table processor_delay.t1 not exists for 1-th check, retry later table partition_table.t2 not exists for 3-th check, retry later table processor_delay.t1 not exists for 2-th check, retry later table partition_table.finish_mark not exists for 15-th check, retry later table ddl_manager.finish_mark not exists for 90-th check, retry later table ddl_attributes.finish_mark not exists for 7-th check, retry later check diff failed 3-th time, retry later table partition_table.t2 not exists for 4-th check, retry later table processor_delay.t1 exists table processor_delay.t2 not exists for 1-th check, retry later table ddl_manager.finish_mark not exists for 91-th check, retry later table ddl_attributes.finish_mark not exists for 8-th check, retry later check diff failed 4-th time, retry later table partition_table.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table partition_table.t2 not exists for 5-th check, retry later table processor_delay.t2 not exists for 2-th check, retry later table ddl_manager.finish_mark not exists for 92-th check, retry later table ddl_attributes.finish_mark not exists for 9-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 12:59:52 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>> check diff failed 5-th time, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/event_filter/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... \033[0;36m<<< Run all test success >>>\033[0m table partition_table.t2 not exists for 6-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache table processor_delay.t2 not exists for 3-th check, retry later [Pipeline] } table ddl_manager.finish_mark exists check diff successfully [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } table ddl_attributes.finish_mark not exists for 10-th check, retry later [Pipeline] // container check_changefeed_state http://127.0.0.1:2379 0c386be2-bec9-4892-9788-c5f781ad1899 finished null + endpoints=http://127.0.0.1:2379 + changefeed_id=0c386be2-bec9-4892-9788-c5f781ad1899 + expected_state=finished + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 0c386be2-bec9-4892-9788-c5f781ad1899 -s + info='{ "upstream_id": 7365375290370071021, "namespace": "default", "id": "0c386be2-bec9-4892-9788-c5f781ad1899", "state": "finished", "checkpoint_tso": 449546859570528261, "checkpoint_time": "2024-05-05 12:59:35.974", "error": null }' + echo '{ "upstream_id": 7365375290370071021, "namespace": "default", "id": "0c386be2-bec9-4892-9788-c5f781ad1899", "state": "finished", "checkpoint_tso": 449546859570528261, "checkpoint_time": "2024-05-05 12:59:35.974", "error": null }' { "upstream_id": 7365375290370071021, "namespace": "default", "id": "0c386be2-bec9-4892-9788-c5f781ad1899", "state": "finished", "checkpoint_tso": 449546859570528261, "checkpoint_time": "2024-05-05 12:59:35.974", "error": null } ++ echo '{' '"upstream_id":' 7365375290370071021, '"namespace":' '"default",' '"id":' '"0c386be2-bec9-4892-9788-c5f781ad1899",' '"state":' '"finished",' '"checkpoint_tso":' 449546859570528261, '"checkpoint_time":' '"2024-05-05' '12:59:35.974",' '"error":' null '}' ++ jq -r .state + state=finished + [[ ! finished == \f\i\n\i\s\h\e\d ]] ++ echo '{' '"upstream_id":' 7365375290370071021, '"namespace":' '"default",' '"id":' '"0c386be2-bec9-4892-9788-c5f781ad1899",' '"state":' '"finished",' '"checkpoint_tso":' 449546859570528261, '"checkpoint_time":' '"2024-05-05' '12:59:35.974",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:59:48 CST 2024] <<<<<< run test case changefeed_finish success! >>>>>> [Pipeline] } [Pipeline] // withEnv [Pipeline] } wait process cdc.test exit for 1-th time... [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } check diff failed 6-th time, retry later wait process cdc.test exit for 2-th time... table partition_table.t2 not exists for 7-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:59:55 CST 2024] <<<<<< run test case ddl_manager success! >>>>>> table processor_delay.t2 not exists for 4-th check, retry later table ddl_attributes.finish_mark exists check diff successfully check diff failed 7-th time, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/resourcecontrol/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table processor_delay.t2 not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 12:59:57 CST 2024] <<<<<< run test case ddl_attributes success! >>>>>> table partition_table.t2 not exists for 8-th check, retry later check diff failed 8-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/resourcecontrol Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... table partition_table.t2 not exists for 9-th check, retry later Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table processor_delay.t2 exists table processor_delay.t3 exists table processor_delay.t4 not exists for 1-th check, retry later check diff failed 9-th time, retry later Verifying downstream PD is started... start tidb cluster in /tmp/tidb_cdc_test/event_filter Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table partition_table.t2 not exists for 10-th check, retry later table processor_delay.t4 not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/force_replicate_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... check diff successfully wait process cdc.test exit for 1-th time... table partition_table.t2 not exists for 11-th check, retry later table processor_delay.t4 not exists for 3-th check, retry later Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release wait process cdc.test exit for 2-th time... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:00:04 CST 2024] <<<<<< run test case changefeed_pause_resume success! >>>>>> table partition_table.t2 not exists for 12-th check, retry later table processor_delay.t4 not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } start tidb cluster in /tmp/tidb_cdc_test/force_replicate_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... [Pipeline] // stage table processor_delay.t4 not exists for 5-th check, retry later [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/multi_source/run.sh using Sink-Type: storage... <<================= [Sun May 5 13:00:07 CST 2024] <<<<<< run test case multi_source success! >>>>>> [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } table partition_table.t2 not exists for 13-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.t2 exists table partition_table.finish_mark not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m table processor_delay.t4 exists table processor_delay.t5 exists table processor_delay.t6 not exists for 1-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release [Pipeline] // dir ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } [Pipeline] // withCredentials [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] } [Pipeline] // cache [Pipeline] // timeout [Pipeline] } [Pipeline] } [Pipeline] // dir [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // withCredentials [Pipeline] // container [Pipeline] } [Pipeline] } table partition_table.finish_mark not exists for 2-th check, retry later [Pipeline] // timeout [Pipeline] // withEnv [Pipeline] } [Pipeline] } table processor_delay.t6 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7ec7500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:10348, start at 2024-05-05 13:00:10.621772966 +0800 CST m=+5.323453980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:10.635 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:10.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:10.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Pipeline] // node [Pipeline] // stage [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] // container [Pipeline] } [Pipeline] } [Pipeline] // withEnv [Pipeline] // withEnv [Pipeline] } [Pipeline] } [Pipeline] // stage [Pipeline] // node [Pipeline] } [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 3-th check, retry later table processor_delay.t6 not exists for 3-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7ec7500014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:10348, start at 2024-05-05 13:00:10.621772966 +0800 CST m=+5.323453980 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:10.635 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:10.630 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:10.630 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7ec7300018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-xj7-5vbfc, pid:10433, start at 2024-05-05 13:00:10.608211617 +0800 CST m=+5.235103080 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:10.616 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:10.622 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:10.622 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/resourcecontrol/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resourcecontrol/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7ef2f80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:6456, start at 2024-05-05 13:00:13.412919787 +0800 CST m=+7.203543489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:13.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:13.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:13.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7ef2f80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:6456, start at 2024-05-05 13:00:13.412919787 +0800 CST m=+7.203543489 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:13.419 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:13.425 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:13.425 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7ed454000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:6544, start at 2024-05-05 13:00:11.427105234 +0800 CST m=+5.165146703 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:11.433 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:11.413 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:11.413 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/event_filter/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/event_filter/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/event_filter/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table partition_table.finish_mark not exists for 4-th check, retry later table processor_delay.t6 not exists for 4-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11746.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) [Sun May 5 13:00:16 CST 2024] <<<<<< START cdc server in event_filter case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.79247926.out server --log-file /tmp/tidb_cdc_test/event_filter/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/event_filter/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + tso='449546870057861121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546870057861121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 13:00:17 CST 2024] <<<<<< START cdc server in resourcecontrol case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.1178211784.out server --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resourcecontrol/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table partition_table.finish_mark not exists for 5-th check, retry later table processor_delay.t6 not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7f38640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw, pid:4235, start at 2024-05-05 13:00:17.857250216 +0800 CST m=+5.150541879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:17.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:17.867 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:17.867 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7f38640014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw, pid:4235, start at 2024-05-05 13:00:17.857250216 +0800 CST m=+5.150541879 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:17.863 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:17.867 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:17.867 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c7f39e40019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-kkz-pwqpw, pid:4320, start at 2024-05-05 13:00:17.962413184 +0800 CST m=+5.199513633 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:17.968 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:17.962 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:17.962 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/force_replicate_table/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/force_replicate_table/tiflash/log/proxy.log"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/cli_with_auth/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:00:19 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fdaa9862-b084-46d8-934c-68ea146da4a9 {"id":"fdaa9862-b084-46d8-934c-68ea146da4a9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885216} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471f8ce00d fdaa9862-b084-46d8-934c-68ea146da4a9 /tidb/cdc/default/default/upstream/7365375866318892108 {"id":7365375866318892108,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fdaa9862-b084-46d8-934c-68ea146da4a9 {"id":"fdaa9862-b084-46d8-934c-68ea146da4a9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885216} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471f8ce00d fdaa9862-b084-46d8-934c-68ea146da4a9 /tidb/cdc/default/default/upstream/7365375866318892108 {"id":7365375866318892108,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fdaa9862-b084-46d8-934c-68ea146da4a9 {"id":"fdaa9862-b084-46d8-934c-68ea146da4a9","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885216} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471f8ce00d fdaa9862-b084-46d8-934c-68ea146da4a9 /tidb/cdc/default/default/upstream/7365375866318892108 {"id":7365375866318892108,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.event_filter.cli.7984.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/event_filter/storage_test/ticdc-event-filter-25586?protocol=canal-json&enable-tidb-extension=true' --server=127.0.0.1:8300 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/event_filter/conf/cf.toml table partition_table.finish_mark not exists for 6-th check, retry later table processor_delay.t6 exists table processor_delay.t7 exists table processor_delay.t8 not exists for 1-th check, retry later Create changefeed successfully! ID: 60ae069b-855c-4948-b3b2-7c465b65acf3 Info: {"upstream_id":7365375866318892108,"namespace":"default","id":"60ae069b-855c-4948-b3b2-7c465b65acf3","sink_uri":"file:///tmp/tidb_cdc_test/event_filter/storage_test/ticdc-event-filter-25586?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T13:00:20.07383705+08:00","start_ts":449546871105126403,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["event_filter.*"],"event_filters":[{"matcher":["event_filter.t1"],"ignore_event":["drop table","delete"],"ignore_sql":null,"ignore_insert_value_expr":"id = 2 or city = 'tokyo'","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_truncate"],"ignore_event":["truncate table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""},{"matcher":["event_filter.t_alter"],"ignore_event":["alter table"],"ignore_sql":null,"ignore_insert_value_expr":"","ignore_update_new_value_expr":"","ignore_update_old_value_expr":"","ignore_delete_value_expr":""}]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546871105126403,"checkpoint_ts":449546871105126403,"checkpoint_time":"2024-05-05 13:00:19.975"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:00:20 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1595ff6c-c95c-4dbf-8ac2-5514e0c92f3c {"id":"1595ff6c-c95c-4dbf-8ac2-5514e0c92f3c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885217} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471f873ad8 1595ff6c-c95c-4dbf-8ac2-5514e0c92f3c /tidb/cdc/default/default/upstream/7365375852308044579 {"id":7365375852308044579,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1595ff6c-c95c-4dbf-8ac2-5514e0c92f3c {"id":"1595ff6c-c95c-4dbf-8ac2-5514e0c92f3c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885217} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471f873ad8 1595ff6c-c95c-4dbf-8ac2-5514e0c92f3c /tidb/cdc/default/default/upstream/7365375852308044579 {"id":7365375852308044579,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/1595ff6c-c95c-4dbf-8ac2-5514e0c92f3c {"id":"1595ff6c-c95c-4dbf-8ac2-5514e0c92f3c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885217} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471f873ad8 1595ff6c-c95c-4dbf-8ac2-5514e0c92f3c /tidb/cdc/default/default/upstream/7365375852308044579 {"id":7365375852308044579,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resourcecontrol.cli.11837.out cli changefeed create --start-ts=449546870057861121 '--sink-uri=file:///tmp/tidb_cdc_test/resourcecontrol/storage_test/ticdc-resourcecontrol-test-1720?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: cc78882a-3d2a-4b22-a0eb-8e3bac774dd6 Info: {"upstream_id":7365375852308044579,"namespace":"default","id":"cc78882a-3d2a-4b22-a0eb-8e3bac774dd6","sink_uri":"file:///tmp/tidb_cdc_test/resourcecontrol/storage_test/ticdc-resourcecontrol-test-1720?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T13:00:20.947459019+08:00","start_ts":449546870057861121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546870057861121,"checkpoint_ts":449546870057861121,"checkpoint_time":"2024-05-05 13:00:15.980"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... [Sun May 5 13:00:21 CST 2024] <<<<<< START cdc server in force_replicate_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.force_replicate_table.56065608.out server --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/force_replicate_table/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + set +x + workdir=/tmp/tidb_cdc_test/event_filter + sink_uri='file:///tmp/tidb_cdc_test/event_filter/storage_test/ticdc-event-filter-25586?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/event_filter ++ date + echo '[Sun May 5 13:00:21 CST 2024] <<<<<< START storage consumer in event_filter case >>>>>>' [Sun May 5 13:00:21 CST 2024] <<<<<< START storage consumer in event_filter case >>>>>> + cd /tmp/tidb_cdc_test/event_filter + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/event_filter + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/event_filter/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/event_filter/storage_test/ticdc-event-filter-25586?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table partition_table.finish_mark not exists for 7-th check, retry later table processor_delay.t8 not exists for 2-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/cli_with_auth Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table event_filter.t1 does not exists table event_filter.t1 not exists for 1-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/resourcecontrol + sink_uri='file:///tmp/tidb_cdc_test/resourcecontrol/storage_test/ticdc-resourcecontrol-test-1720?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/resourcecontrol ++ date + echo '[Sun May 5 13:00:22 CST 2024] <<<<<< START storage consumer in resourcecontrol case >>>>>>' [Sun May 5 13:00:22 CST 2024] <<<<<< START storage consumer in resourcecontrol case >>>>>> + cd /tmp/tidb_cdc_test/resourcecontrol + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/resourcecontrol + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/resourcecontrol/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/resourcecontrol/storage_test/ticdc-resourcecontrol-test-1720?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table resourcecontrol.finish_mark not exists for 1-th check, retry later table partition_table.finish_mark not exists for 8-th check, retry later table processor_delay.t8 not exists for 3-th check, retry later table event_filter.t1 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:00:24 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0943cdfa-138e-4882-aa99-e01b08cf8731 {"id":"0943cdfa-138e-4882-aa99-e01b08cf8731","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885221} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471fa44ac7 0943cdfa-138e-4882-aa99-e01b08cf8731 /tidb/cdc/default/default/upstream/7365375892017875482 {"id":7365375892017875482,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0943cdfa-138e-4882-aa99-e01b08cf8731 {"id":"0943cdfa-138e-4882-aa99-e01b08cf8731","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885221} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471fa44ac7 0943cdfa-138e-4882-aa99-e01b08cf8731 /tidb/cdc/default/default/upstream/7365375892017875482 {"id":7365375892017875482,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/0943cdfa-138e-4882-aa99-e01b08cf8731 {"id":"0943cdfa-138e-4882-aa99-e01b08cf8731","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885221} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471fa44ac7 0943cdfa-138e-4882-aa99-e01b08cf8731 /tidb/cdc/default/default/upstream/7365375892017875482 {"id":7365375892017875482,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: dfee17f7-8aeb-4cfa-ae0b-c7f6b53d27ca Info: {"upstream_id":7365375892017875482,"namespace":"default","id":"dfee17f7-8aeb-4cfa-ae0b-c7f6b53d27ca","sink_uri":"file:///tmp/tidb_cdc_test/force_replicate_table/storage_test/ticdc-force_replicate_table-test-4586?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T13:00:24.619300454+08:00","start_ts":449546871391649793,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":true,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546871391649793,"checkpoint_ts":449546871391649793,"checkpoint_time":"2024-05-05 13:00:21.068"} + workdir=/tmp/tidb_cdc_test/force_replicate_table + sink_uri='file:///tmp/tidb_cdc_test/force_replicate_table/storage_test/ticdc-force_replicate_table-test-4586?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/force_replicate_table ++ date + echo '[Sun May 5 13:00:24 CST 2024] <<<<<< START storage consumer in force_replicate_table case >>>>>>' [Sun May 5 13:00:24 CST 2024] <<<<<< START storage consumer in force_replicate_table case >>>>>> + cd /tmp/tidb_cdc_test/force_replicate_table + '[' /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml '!=' '' ']' + cd /tmp/tidb_cdc_test/force_replicate_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/force_replicate_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/force_replicate_table/storage_test/ticdc-force_replicate_table-test-4586?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' --config /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/force_replicate_table/conf/changefeed.toml table resourcecontrol.finish_mark not exists for 2-th check, retry later table partition_table.finish_mark not exists for 9-th check, retry later table processor_delay.t8 not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t1 not exists for 3-th check, retry later table force_replicate_table.t0 not exists for 1-th check, retry later table resourcecontrol.finish_mark not exists for 3-th check, retry later table partition_table.finish_mark not exists for 10-th check, retry later table processor_delay.t8 not exists for 5-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t1 not exists for 4-th check, retry later table force_replicate_table.t0 not exists for 2-th check, retry later table resourcecontrol.finish_mark not exists for 4-th check, retry later table partition_table.finish_mark not exists for 11-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table event_filter.t1 not exists for 5-th check, retry later table force_replicate_table.t0 not exists for 3-th check, retry later table processor_delay.t8 not exists for 6-th check, retry later table resourcecontrol.finish_mark not exists for 5-th check, retry later table partition_table.finish_mark not exists for 12-th check, retry later table event_filter.t1 exists table event_filter.t_normal exists table event_filter.t_truncate not exists for 1-th check, retry later table force_replicate_table.t0 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t8 exists table processor_delay.t9 exists table processor_delay.t10 not exists for 1-th check, retry later table resourcecontrol.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... table partition_table.finish_mark not exists for 13-th check, retry later wait process cdc.test exit for 2-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c800ca80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:12156, start at 2024-05-05 13:00:31.434662486 +0800 CST m=+5.421370348 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:31.442 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:31.402 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:31.402 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c800ca80018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:12156, start at 2024-05-05 13:00:31.434662486 +0800 CST m=+5.421370348 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:31.442 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:31.402 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:31.402 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c800d2c000d Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:12237, start at 2024-05-05 13:00:31.447712754 +0800 CST m=+5.384543179 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:02:31.454 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:00:31.435 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:50:31.435 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/cli_with_auth/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli_with_auth/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:00:34 CST 2024] <<<<<< run test case resourcecontrol success! >>>>>> table event_filter.t_truncate exists table event_filter.t_alter not exists for 1-th check, retry later table force_replicate_table.t0 not exists for 5-th check, retry later table processor_delay.t10 not exists for 2-th check, retry later table partition_table.finish_mark not exists for 14-th check, retry later table event_filter.t_alter not exists for 2-th check, retry later table force_replicate_table.t0 exists table force_replicate_table.t1 exists table force_replicate_table.t2 exists table force_replicate_table.t3 exists table force_replicate_table.t4 exists table force_replicate_table.t5 not exists for 1-th check, retry later [2024/05/05 13:00:28.918 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/05 13:00:28.918 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365375291974706174] [2024/05/05 13:00:28.919 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/05 13:00:28.919 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/05 13:00:28.919 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/05 13:00:28.919 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/05 13:00:28.920 +08:00] [INFO] [pd_service_discovery.go:1016] ["[pd] switch leader"] [new-leader=http://127.0.0.1:2379] [old-leader=] [2024/05/05 13:00:28.920 +08:00] [INFO] [pd_service_discovery.go:498] ["[pd] init cluster id"] [cluster-id=7365375291974706174] [2024/05/05 13:00:28.921 +08:00] [INFO] [client.go:606] ["[pd] changing service mode"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/05 13:00:28.921 +08:00] [INFO] [tso_client.go:236] ["[tso] switch dc tso global allocator serving url"] [dc-location=global] [new-url=http://127.0.0.1:2379] [2024/05/05 13:00:28.921 +08:00] [INFO] [tso_dispatcher.go:359] ["[tso] tso dispatcher created"] [dc-location=global] [2024/05/05 13:00:28.921 +08:00] [INFO] [client.go:612] ["[pd] service mode changed"] [old-mode=UNKNOWN_SVC_MODE] [new-mode=PD_SVC_MODE] [2024/05/05 13:00:28.922 +08:00] [INFO] [tikv_driver.go:197] ["using API V1."] [2024/05/05 13:00:28.922 +08:00] [INFO] [main.go:180] ["genLock started"] [2024/05/05 13:00:28.923 +08:00] [INFO] [store_cache.go:477] ["change store resolve state"] [store=3] [addr=127.0.0.1:20161] [from=unresolved] [to=resolved] [liveness-state=reachable] table processor_delay.t10 not exists for 3-th check, retry later + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13513.out cli tso query --pd=http://127.0.0.1:2379 table partition_table.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... + set +x + tso='449546875516223489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546875516223489 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table event_filter.t_alter not exists for 3-th check, retry later table force_replicate_table.t5 not exists for 2-th check, retry later table processor_delay.t10 not exists for 4-th check, retry later wait process cdc.test exit for 2-th time... [Sun May 5 13:00:39 CST 2024] <<<<<< START cdc server in cli_with_auth case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.1356713569.out server --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli_with_auth/cdc_data --cluster-id default --config /tmp/tidb_cdc_test/cli_with_auth/server.toml + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 [2024/05/05 13:00:38.929 +08:00] [INFO] [main.go:196] ["genLock done"] [2024/05/05 13:00:38.930 +08:00] [INFO] [resource_manager_client.go:295] ["[resource manager] exit resource token dispatcher"] [2024/05/05 13:00:38.930 +08:00] [INFO] [pd_service_discovery.go:550] ["[pd] exit member loop due to context canceled"] [2024/05/05 13:00:38.930 +08:00] [INFO] [tso_client.go:140] ["closing tso client"] [2024/05/05 13:00:38.930 +08:00] [INFO] [tso_dispatcher.go:268] ["exit tso dispatcher loop"] [2024/05/05 13:00:38.930 +08:00] [INFO] [tso_dispatcher.go:455] ["[tso] stop fetching the pending tso requests due to context canceled"] [dc-location=global] [2024/05/05 13:00:38.930 +08:00] [INFO] [tso_dispatcher.go:214] ["exit tso requests cancel loop"] [2024/05/05 13:00:38.930 +08:00] [INFO] [tso_dispatcher.go:380] ["[tso] exit tso dispatcher"] [dc-location=global] [2024/05/05 13:00:38.930 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/05 13:00:38.930 +08:00] [INFO] [tso_client.go:145] ["close tso client"] [2024/05/05 13:00:38.930 +08:00] [INFO] [tso_batch_controller.go:158] ["[pd] clear the tso batch controller"] [max-batch-size=10000] [best-batch-size=1] [collected-request-count=0] [pending-request-count=0] [2024/05/05 13:00:38.930 +08:00] [INFO] [tso_client.go:155] ["tso client is closed"] [2024/05/05 13:00:38.930 +08:00] [INFO] [pd_service_discovery.go:637] ["[pd] close pd service discovery client"] cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:00:39 CST 2024] <<<<<< run test case partition_table success! >>>>>> table event_filter.t_alter not exists for 4-th check, retry later table force_replicate_table.t5 not exists for 3-th check, retry later table processor_delay.t10 exists table processor_delay.t11 exists table processor_delay.t12 not exists for 1-th check, retry later pass check, checkpoint tso not forward after 10s run task successfully wait process 8405 exit for 1-th time... wait process 8405 exit for 2-th time... wait process 8405 exit for 3-th time... wait process 8405 exit for 4-th time... /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/_utils/kill_cdc_pid: line 19: kill: (8405) - No such process wait process 8405 exit for 5-th time... process 8405 already exit [Sun May 5 13:00:34 CST 2024] <<<<<< START cdc server in ddl_only_block_related_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_only_block_related_table.92189220.out server --log-file /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_only_block_related_table/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:00:37 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365375559109997918 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:59:09.698740728 +0800 CST StartTs:449546852657266693 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004430000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546852670111752} {CheckpointTs:449546857782181901 MinTableBarrierTs:449546875568390145 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2317a4a-4693-48e5-ba1d-46d0cd68965c {"id":"d2317a4a-4693-48e5-ba1d-46d0cd68965c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e7614cd d2317a4a-4693-48e5-ba1d-46d0cd68965c /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365375559109997918,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:59:09.698740728+08:00","start-ts":449546852657266693,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546852670111752} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449546857782181901,"min-table-barrier-ts":449546875568390145,"admin-job-type":0} /tidb/cdc/default/default/task/position/d2317a4a-4693-48e5-ba1d-46d0cd68965c/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375559109997918 {"id":7365375559109997918,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365375559109997918 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:59:09.698740728 +0800 CST StartTs:449546852657266693 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004430000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546852670111752} {CheckpointTs:449546857782181901 MinTableBarrierTs:449546875568390145 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2317a4a-4693-48e5-ba1d-46d0cd68965c {"id":"d2317a4a-4693-48e5-ba1d-46d0cd68965c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e7614cd d2317a4a-4693-48e5-ba1d-46d0cd68965c /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365375559109997918,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:59:09.698740728+08:00","start-ts":449546852657266693,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synce+ grep -q 'failed to get info:' d-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546852670111752} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449546857782181901,"min-table-barrier-ts":449546875568390145,"admin-job-type":0} /tidb/cdc/default/default/task/position/d2317a4a-4693-48e5-ba1d-46d0cd68965c/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375559109997918 {"id":7365375559109997918,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** owner info ***: *** processors info ***: changefeedID: default/ddl-only-block-related-table {UpstreamID:7365375559109997918 Namespace:default ID:ddl-only-block-related-table SinkURI:file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 12:59:09.698740728 +0800 CST StartTs:449546852657266693 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc004430000 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546852670111752} {CheckpointTs:449546857782181901 MinTableBarrierTs:449546875568390145 AdminJobType:noop} span: {table_id:110,start_key:7480000000000000ff6e5f720000000000fa,end_key:7480000000000000ff6e5f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing span: {table_id:112,start_key:7480000000000000ff705f720000000000fa,end_key:7480000000000000ff705f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing span: {table_id:106,start_key:7480000000000000ff6a5f720000000000fa,end_key:7480000000000000ff6a5f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing span: {table_id:108,start_key:7480000000000000ff6c5f720000000000fa,end_key:7480000000000000ff6c5f730000000000fa}, resolvedTs: 449546857782181901, checkpointTs: 449546857782181901, state: Preparing *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d2317a4a-4693-48e5-ba1d-46d0cd68965c {"id":"d2317a4a-4693-48e5-ba1d-46d0cd68965c","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885234} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471e7614cd d2317a4a-4693-48e5-ba1d-46d0cd68965c /tidb/cdc/default/default/changefeed/info/ddl-only-block-related-table {"upstream-id":7365375559109997918,"namespace":"default","changefeed-id":"ddl-only-block-related-table","sink-uri":"file:///tmp/tidb_cdc_test/ddl_only_block_related_table/storage_test/ticdc-common-1-test-30262?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T12:59:09.698740728+08:00","start-ts":449546852657266693,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546852670111752} /tidb/cdc/default/default/changefeed/status/ddl-only-block-related-table {"checkpoint-ts":449546857782181901,"min-table-barrier-ts":449546875568390145,"admin-job-type":0} /tidb/cdc/default/default/task/position/d2317a4a-4693-48e5-ba1d-46d0cd68965c/ddl-only-block-related-table {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365375559109997918 {"id":7365375559109997918,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x check diff failed 1-th time, retry later check diff failed 2-th time, retry later check diff failed 3-th time, retry later table event_filter.t_alter exists table force_replicate_table.t5 not exists for 4-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:00:42 GMT < Content-Length: 861 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ad2af90-a05f-4260-88cb-99f4292d0052 {"id":"4ad2af90-a05f-4260-88cb-99f4292d0052","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471fe1e74c 4ad2af90-a05f-4260-88cb-99f4292d0052 /tidb/cdc/default/default/upstream/7365375948894311395 {"id":7365375948894311395,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ad2af90-a05f-4260-88cb-99f4292d0052 {"id":"4ad2af90-a05f-4260-88cb-99f4292d0052","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471fe1e74c 4ad2af90-a05f-4260-88cb-99f4292d0052 /tidb/cdc/default/default/upstream/7365375948894311395 {"id":7365375948894311395,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4ad2af90-a05f-4260-88cb-99f4292d0052 {"id":"4ad2af90-a05f-4260-88cb-99f4292d0052","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885239} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f471fe1e74c 4ad2af90-a05f-4260-88cb-99f4292d0052 /tidb/cdc/default/default/upstream/7365375948894311395 {"id":7365375948894311395,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.13625.out cli changefeed create --start-ts=449546875516223489 '--sink-uri=file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-30417?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365375948894311395,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-30417?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T13:00:42.855461179+08:00","start_ts":449546875516223489,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546875516223489,"checkpoint_ts":449546875516223489,"checkpoint_time":"2024-05-05 13:00:36.802"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table event_filter.finish_mark exists table processor_delay.t12 not exists for 2-th check, retry later check diff failed 1-th time, retry later check diff successfully check_ts_forward ddl-only-block-related-table + set +x + workdir=/tmp/tidb_cdc_test/cli_with_auth + sink_uri='file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-30417?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/cli_with_auth ++ date + echo '[Sun May 5 13:00:44 CST 2024] <<<<<< START storage consumer in cli_with_auth case >>>>>>' [Sun May 5 13:00:44 CST 2024] <<<<<< START storage consumer in cli_with_auth case >>>>>> + cd /tmp/tidb_cdc_test/cli_with_auth + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/cli_with_auth + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/cli_with_auth/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-30417?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table test.simple not exists for 1-th check, retry later table force_replicate_table.t5 not exists for 5-th check, retry later table processor_delay.t12 not exists for 3-th check, retry later table test.t2 not exists for 1-th check, retry later check diff failed 2-th time, retry later changefeed is working normally rts: 449546877141254140->449546877665542140 checkpoint: 449546877141254140->449546877665542140 run task successfully wait process cdc.test exit for 1-th time... table force_replicate_table.t5 exists table force_replicate_table.t6 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) wait process cdc.test exit for 2-th time... table test.t2 not exists for 2-th check, retry later table test.simple not exists for 2-th check, retry later [Pipeline] // cache [Pipeline] } table processor_delay.t12 not exists for 4-th check, retry later [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } wait process cdc.test exit for 3-th time... [Pipeline] // withEnv check diff failed 3-th time, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 13:00:47 CST 2024] <<<<<< run test case ddl_only_block_related_table success! >>>>>> [Pipeline] } [Pipeline] // node [Pipeline] } table force_replicate_table.t6 not exists for 2-th check, retry later [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage table test.simple not exists for 3-th check, retry later [Pipeline] } table processor_delay.t12 not exists for 5-th check, retry later table test.t2 not exists for 3-th check, retry later check diff failed 4-th time, retry later table force_replicate_table.t6 not exists for 3-th check, retry later table test.simple not exists for 4-th check, retry later table processor_delay.t12 exists table processor_delay.t13 exists table processor_delay.t14 not exists for 1-th check, retry later table test.t2 not exists for 4-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } check diff failed 5-th time, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } table test.simple not exists for 5-th check, retry later [Pipeline] // stage [Pipeline] } table force_replicate_table.t6 not exists for 4-th check, retry later table test.t2 not exists for 5-th check, retry later table processor_delay.t14 not exists for 2-th check, retry later check diff failed 6-th time, retry later table test.simple not exists for 6-th check, retry later table force_replicate_table.t6 not exists for 5-th check, retry later table test.t2 not exists for 6-th check, retry later table processor_delay.t14 not exists for 3-th check, retry later check diff failed 7-th time, retry later table force_replicate_table.t6 exists check_data_subset force_replicate_table.t0 127.0.0.1 4000 127.0.0.1 3306 table test.simple exists table test.`simple-dash` exists + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546880562495496, "checkpoint_time": "2024-05-05 13:00:56.052", "error": null }' + echo '{ "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546880562495496, "checkpoint_time": "2024-05-05 13:00:56.052", "error": null }' { "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546880562495496, "checkpoint_time": "2024-05-05 13:00:56.052", "error": null } ++ echo '{' '"upstream_id":' 7365375948894311395, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546880562495496, '"checkpoint_time":' '"2024-05-05' '13:00:56.052",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365375948894311395, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546880562495496, '"checkpoint_time":' '"2024-05-05' '13:00:56.052",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379 table test.t2 not exists for 7-th check, retry later table processor_delay.t14 not exists for 4-th check, retry later changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679 run task successfully check_data_subset force_replicate_table.t1 127.0.0.1 4000 127.0.0.1 3306 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779 changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 run task successfully check_data_subset force_replicate_table.t2 127.0.0.1 4000 127.0.0.1 3306 Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed update changefeed config should fail when changefeed is running, got Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc0017d3640} {Type:update Path:[Config SyncPointRetention] From: To:0xc0017d3648} {Type:update Path:[Config Consistent] From: To:0xc00127e850} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14049.out cli changefeed --changefeed-id custom-changefeed-name pause check diff failed 8-th time, retry later run task successfully check_data_subset force_replicate_table.t3 127.0.0.1 4000 127.0.0.1 3306 table processor_delay.t14 not exists for 5-th check, retry later PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/... run task successfully check_data_subset force_replicate_table.t4 127.0.0.1 4000 127.0.0.1 3306 table test.t2 not exists for 8-th check, retry later run task successfully check_data_subset force_replicate_table.t5 127.0.0.1 4000 127.0.0.1 3306 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/changefeed_auto_stop/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + set +x check diff failed 9-th time, retry later table processor_delay.t14 exists table processor_delay.t15 exists table processor_delay.t16 not exists for 1-th check, retry later run task successfully check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=7,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 1-th time, retry later table test.t2 not exists for 9-th check, retry later check diff successfully start tidb cluster in /tmp/tidb_cdc_test/changefeed_auto_stop Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=7,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 2-th time, retry later wait process cdc.test exit for 1-th time... table processor_delay.t16 not exists for 2-th check, retry later table test.t2 not exists for 10-th check, retry later wait process cdc.test exit for 2-th time... + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=stopped + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:01:03 CST 2024] <<<<<< run test case event_filter success! >>>>>> + info='{ "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449546881086783495, "checkpoint_time": "2024-05-05 13:00:58.052", "error": null }' + echo '{ "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449546881086783495, "checkpoint_time": "2024-05-05 13:00:58.052", "error": null }' { "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "stopped", "checkpoint_tso": 449546881086783495, "checkpoint_time": "2024-05-05 13:00:58.052", "error": null } ++ echo '{' '"upstream_id":' 7365375948894311395, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449546881086783495, '"checkpoint_time":' '"2024-05-05' '13:00:58.052",' '"error":' null '}' ++ jq -r .state + state=stopped + [[ ! stopped == \s\t\o\p\p\e\d ]] ++ echo '{' '"upstream_id":' 7365375948894311395, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449546881086783495, '"checkpoint_time":' '"2024-05-05' '13:00:58.052",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14135.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli_with_auth/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name Diff of changefeed config: {Type:update Path:[Config CaseSensitive] From:false To:true} {Type:update Path:[Config SyncPointInterval] From: To:0xc00185d468} {Type:update Path:[Config SyncPointRetention] From: To:0xc00185d4a8} {Type:update Path:[Config Consistent] From: To:0xc001ac4e00} {Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true} Update changefeed config successfully! ID: custom-changefeed-name Info: {"upstream_id":7365375948894311395,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-30417?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T13:00:42.855461179+08:00","start_ts":449546875516223489,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_table_monitor":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"delete_only_output_handle_key_columns":null,"content_compatible":null,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":0,"checkpoint_ts":449546881086783495,"checkpoint_time":"2024-05-05 13:00:58.052"} PASS table processor_delay.t16 not exists for 3-th check, retry later table test.t2 not exists for 11-th check, retry later coverage: 2.8% of statements in github.com/pingcap/tiflow/... + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14174.out cli changefeed --changefeed-id custom-changefeed-name resume check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=7,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 3-th time, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t16 not exists for 4-th check, retry later table test.t2 not exists for 12-th check, retry later PASS coverage: 2.1% of statements in github.com/pingcap/tiflow/... Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + set +x table processor_delay.t16 not exists for 5-th check, retry later table test.t2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... table processor_delay.t16 exists table processor_delay.t17 exists table processor_delay.t18 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 13:01:11 CST 2024] <<<<<< run test case resolve_lock success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546884507762690, "checkpoint_time": "2024-05-05 13:01:11.102", "error": null }' + echo '{ "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546884507762690, "checkpoint_time": "2024-05-05 13:01:11.102", "error": null }' { "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546884507762690, "checkpoint_time": "2024-05-05 13:01:11.102", "error": null } ++ echo '{' '"upstream_id":' 7365375948894311395, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546884507762690, '"checkpoint_time":' '"2024-05-05' '13:01:11.102",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365375948894311395, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546884507762690, '"checkpoint_time":' '"2024-05-05' '13:01:11.102",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14264.out cli changefeed --changefeed-id custom-changefeed-name remove Changefeed remove successfully. ID: custom-changefeed-name CheckpointTs: 449546884507762690 SinkURI: file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-30417?protocol=canal-json&enable-tidb-extension=true PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 id=7,a=NULL doesn't exist in downstream table force_replicate_table.t6 run task failed 4-th time, retry later table processor_delay.t18 not exists for 2-th check, retry later + set +x ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t18 not exists for 3-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sql_mode/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/sql_mode + CDC_BINARY=cdc.test + SINK_TYPE=storage + CDC_COUNT=3 + DB_COUNT=4 + rm -rf /tmp/tidb_cdc_test/sql_mode + mkdir -p /tmp/tidb_cdc_test/sql_mode + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c82a5080014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:10546, start at 2024-05-05 13:01:13.949652034 +0800 CST m=+5.239735666 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:13.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:13.922 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:13.922 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c82a5080014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:10546, start at 2024-05-05 13:01:13.949652034 +0800 CST m=+5.239735666 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:13.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:13.922 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:13.922 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c82a5b00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-69h-17zn3, pid:10622, start at 2024-05-05 13:01:13.999936386 +0800 CST m=+5.225904342 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:14.009 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:14.013 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:14.013 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored table processor_delay.t18 not exists for 4-th check, retry later Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/changefeed_auto_stop/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14345.out cli changefeed create '--sink-uri=file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-30417?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name [WARN] --tz is deprecated in changefeed settings. Create changefeed successfully! ID: custom-changefeed-name Info: {"upstream_id":7365375948894311395,"namespace":"default","id":"custom-changefeed-name","sink_uri":"file:///tmp/tidb_cdc_test/cli_with_auth/storage_test/ticdc-cli-test-30417?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T13:01:17.703126763+08:00","start_ts":449546886198853638,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546886198853638,"checkpoint_ts":449546886198853638,"checkpoint_time":"2024-05-05 13:01:17.553"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table processor_delay.t18 not exists for 5-th check, retry later + set +x ***************** properties ***************** "requestdistribution"="uniform" "insertproportion"="0" "mysql.host"="127.0.0.1" "readproportion"="0" "workload"="core" "recordcount"="20" "mysql.port"="4000" "scanproportion"="0" "updateproportion"="0" "threadcount"="4" "readallfields"="true" "operationcount"="0" "mysql.db"="changefeed_auto_stop_1" "mysql.user"="root" "dotransactions"="false" ********************************************** Run finished, takes 10.046847ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3969.7, Avg(us): 1923, Min(us): 1010, Max(us): 5043, 95th(us): 6000, 99th(us): 6000 table processor_delay.t18 exists table processor_delay.t19 exists table processor_delay.t20 not exists for 1-th check, retry later check_data_subset force_replicate_table.t6 127.0.0.1 4000 127.0.0.1 3306 run task successfully ***************** properties ***************** "mysql.host"="127.0.0.1" "updateproportion"="0" "readallfields"="true" "operationcount"="0" "mysql.db"="changefeed_auto_stop_2" "scanproportion"="0" "mysql.user"="root" "dotransactions"="false" "workload"="core" "readproportion"="0" "mysql.port"="4000" "insertproportion"="0" "requestdistribution"="uniform" "recordcount"="20" "threadcount"="4" ********************************************** Run finished, takes 11.318752ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3395.7, Avg(us): 2135, Min(us): 1025, Max(us): 5346, 95th(us): 6000, 99th(us): 6000 wait process cdc.test exit for 1-th time... ***************** properties ***************** "requestdistribution"="uniform" "threadcount"="4" "mysql.host"="127.0.0.1" "mysql.db"="changefeed_auto_stop_3" "updateproportion"="0" "mysql.port"="4000" "readproportion"="0" "operationcount"="0" "dotransactions"="false" "insertproportion"="0" "recordcount"="20" "mysql.user"="root" "readallfields"="true" "workload"="core" "scanproportion"="0" ********************************************** + endpoints=http://127.0.0.1:2379 + changefeed_id=custom-changefeed-name + expected_state=normal + error_msg=null + tls_dir= + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s + info='{ "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546887129202690, "checkpoint_time": "2024-05-05 13:01:21.102", "error": null }' + echo '{ "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546887129202690, "checkpoint_time": "2024-05-05 13:01:21.102", "error": null }' { "upstream_id": 7365375948894311395, "namespace": "default", "id": "custom-changefeed-name", "state": "normal", "checkpoint_tso": 449546887129202690, "checkpoint_time": "2024-05-05 13:01:21.102", "error": null } ++ echo '{' '"upstream_id":' 7365375948894311395, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546887129202690, '"checkpoint_time":' '"2024-05-05' '13:01:21.102",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365375948894311395, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449546887129202690, '"checkpoint_time":' '"2024-05-05' '13:01:21.102",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] wait process cdc.test exit for 2-th time... Run finished, takes 11.312099ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3423.8, Avg(us): 2112, Min(us): 1036, Max(us): 5451, 95th(us): 6000, 99th(us): 6000 start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:01:22 CST 2024] <<<<<< run test case force_replicate_table success! >>>>>> table processor_delay.t20 not exists for 2-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/move_table/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... ***************** properties ***************** "requestdistribution"="uniform" "threadcount"="4" "mysql.user"="root" "insertproportion"="0" "updateproportion"="0" "readallfields"="true" "dotransactions"="false" "readproportion"="0" "scanproportion"="0" "workload"="core" "operationcount"="0" "recordcount"="20" "mysql.host"="127.0.0.1" "mysql.port"="4000" "mysql.db"="changefeed_auto_stop_4" ********************************************** Run finished, takes 11.015351ms INSERT - Takes(s): 0.0, Count: 20, OPS: 3913.1, Avg(us): 2059, Min(us): 879, Max(us): 5793, 95th(us): 6000, 99th(us): 6000 [Sun May 5 13:01:23 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1207712079.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data1 --cluster-id default --addr 127.0.0.1:8301 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14474.out cli unsafe delete-service-gc-safepoint Verifying downstream PD is started... Confirm that you know what this command will do and use it at your own risk [Y/N] CDC service GC safepoint truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table processor_delay.t20 not exists for 3-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14510.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379 Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release reset and all metadata truncated in PD! PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... start tidb cluster in /tmp/tidb_cdc_test/move_table Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:01:26 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfc84192-3a1b-4783-9c80-791cf2158fcd {"id":"cfc84192-3a1b-4783-9c80-791cf2158fcd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885283} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47207d1a49 cfc84192-3a1b-4783-9c80-791cf2158fcd /tidb/cdc/default/default/upstream/7365376133358082772 {"id":7365376133358082772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfc84192-3a1b-4783-9c80-791cf2158fcd {"id":"cfc84192-3a1b-4783-9c80-791cf2158fcd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885283} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47207d1a49 cfc84192-3a1b-4783-9c80-791cf2158fcd /tidb/cdc/default/default/upstream/7365376133358082772 {"id":7365376133358082772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/cfc84192-3a1b-4783-9c80-791cf2158fcd {"id":"cfc84192-3a1b-4783-9c80-791cf2158fcd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885283} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47207d1a49 cfc84192-3a1b-4783-9c80-791cf2158fcd /tidb/cdc/default/default/upstream/7365376133358082772 {"id":7365376133358082772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x [Sun May 5 13:01:26 CST 2024] <<<<<< START cdc server in changefeed_auto_stop case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/pipeline/ProcessorSyncResolvedError=1*return(true);github.com/pingcap/tiflow/cdc/processor/ProcessorUpdatePositionDelaying=sleep(1000)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.changefeed_auto_stop.1213112133.out server --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_data2 --cluster-id default --addr 127.0.0.1:8302 --pd http://127.0.0.1:2379 ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t20 not exists for 4-th check, retry later + set +x Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t20 not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:01:29 GMT < Content-Length: 1275 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cb12af3-6f4c-46c0-a008-9e83f47aaf8e {"id":"7cb12af3-6f4c-46c0-a008-9e83f47aaf8e","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885286} /tidb/cdc/default/__cdc_meta__/capture/cfc84192-3a1b-4783-9c80-791cf2158fcd {"id":"cfc84192-3a1b-4783-9c80-791cf2158fcd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885283} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47207d1a49 cfc84192-3a1b-4783-9c80-791cf2158fcd /tidb/cdc/default/__cdc_meta__/owner/22318f47207d1a80 7cb12af3-6f4c-46c0-a008-9e83f47aaf8e /tidb/cdc/default/default/upstream/7365376133358082772 {"id":7365376133358082772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cb12af3-6f4c-46c0-a008-9e83f47aaf8e {"id":"7cb12af3-6f4c-46c0-a008-9e83f47aaf8e","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885286} /tidb/cdc/default/__cdc_meta__/capture/cfc84192-3a1b-4783-9c80-791cf2158fcd {"id":"cfc84192-3a1b-4783-9c80-791cf2158fcd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885283} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47207d1a49 cfc84192-3a1b-4783-9c80-791cf2158fcd /tidb/cdc/default/__cdc_meta__/owner/22318f47207d1a80 7cb12af3-6f4c-46c0-a008-9e83f47aaf8e /tidb/cdc/default/default/upstream/7365376133358082772 {"id":7365376133358082772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7cb12af3-6f4c-46c0-a008-9e83f47aaf8e {"id":"7cb12af3-6f4c-46c0-a008-9e83f47aaf8e","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885286} /tidb/cdc/default/__cdc_meta__/capture/cfc84192-3a1b-4783-9c80-791cf2158fcd {"id":"cfc84192-3a1b-4783-9c80-791cf2158fcd","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885283} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47207d1a49 cfc84192-3a1b-4783-9c80-791cf2158fcd /tidb/cdc/default/__cdc_meta__/owner/22318f47207d1a80 7cb12af3-6f4c-46c0-a008-9e83f47aaf8e /tidb/cdc/default/default/upstream/7365376133358082772 {"id":7365376133358082772,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/changefeed_auto_stop + sink_uri='file:///tmp/tidb_cdc_test/changefeed_auto_stop/storage_test/ticdc-changefeed-auto-stop-test-13816?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/changefeed_auto_stop ++ date + echo '[Sun May 5 13:01:29 CST 2024] <<<<<< START storage consumer in changefeed_auto_stop case >>>>>>' [Sun May 5 13:01:29 CST 2024] <<<<<< START storage consumer in changefeed_auto_stop case >>>>>> + cd /tmp/tidb_cdc_test/changefeed_auto_stop + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/changefeed_auto_stop + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/changefeed_auto_stop/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/changefeed_auto_stop/storage_test/ticdc-changefeed-auto-stop-test-13816?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_state http://127.0.0.1:2379 08fba91b-6f65-4fd8-a2d8-d3a55c85aac4 normal null + endpoints=http://127.0.0.1:2379 + changefeed_id=08fba91b-6f65-4fd8-a2d8-d3a55c85aac4 + expected_state=normal + error_msg=null + tls_dir=null + [[ http://127.0.0.1:2379 =~ https ]] ++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c 08fba91b-6f65-4fd8-a2d8-d3a55c85aac4 -s + info='{ "upstream_id": 7365376133358082772, "namespace": "default", "id": "08fba91b-6f65-4fd8-a2d8-d3a55c85aac4", "state": "normal", "checkpoint_tso": 449546886597312514, "checkpoint_time": "2024-05-05 13:01:19.073", "error": null }' + echo '{ "upstream_id": 7365376133358082772, "namespace": "default", "id": "08fba91b-6f65-4fd8-a2d8-d3a55c85aac4", "state": "normal", "checkpoint_tso": 449546886597312514, "checkpoint_time": "2024-05-05 13:01:19.073", "error": null }' { "upstream_id": 7365376133358082772, "namespace": "default", "id": "08fba91b-6f65-4fd8-a2d8-d3a55c85aac4", "state": "normal", "checkpoint_tso": 449546886597312514, "checkpoint_time": "2024-05-05 13:01:19.073", "error": null } ++ echo '{' '"upstream_id":' 7365376133358082772, '"namespace":' '"default",' '"id":' '"08fba91b-6f65-4fd8-a2d8-d3a55c85aac4",' '"state":' '"normal",' '"checkpoint_tso":' 449546886597312514, '"checkpoint_time":' '"2024-05-05' '13:01:19.073",' '"error":' null '}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] ++ echo '{' '"upstream_id":' 7365376133358082772, '"namespace":' '"default",' '"id":' '"08fba91b-6f65-4fd8-a2d8-d3a55c85aac4",' '"state":' '"normal",' '"checkpoint_tso":' 449546886597312514, '"checkpoint_time":' '"2024-05-05' '13:01:19.073",' '"error":' null '}' ++ jq -r .error.message + message=null + [[ ! null =~ null ]] run task successfully table changefeed_auto_stop_1.usertable not exists for 1-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14628.out cli unsafe resolve-lock --region=54 PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table processor_delay.t20 exists table processor_delay.t21 exists table processor_delay.t22 not exists for 1-th check, retry later + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli_with_auth.cli.14663.out cli unsafe resolve-lock --region=54 --ts=449546888741388294 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_1.usertable not exists for 2-th check, retry later PASS coverage: 2.0% of statements in github.com/pingcap/tiflow/... table processor_delay.t22 not exists for 2-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ca48000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9509, start at 2024-05-05 13:01:32.706776217 +0800 CST m=+5.047232997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.690 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.690 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ca48000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9509, start at 2024-05-05 13:01:32.706776217 +0800 CST m=+5.047232997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.690 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.690 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83cd440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9586, start at 2024-05-05 13:01:32.882696933 +0800 CST m=+5.162788968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.881 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.881 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sql_mode/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sql_mode/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/db/proxy"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sql_mode/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 218 100 212 100 6 179k 5208 --:--:-- --:--:-- --:--:-- 207k { "error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }", "error_code": "CDC:ErrAPIInvalidParam" ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table changefeed_auto_stop_1.usertable not exists for 3-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 3017 0 --:--:-- --:--:-- --:--:-- 3027 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-05 13:01:21.900","puller_resolved_ts":"2024-05-05 13:01:14.900","last_synced_ts":"2024-05-05 12:59:06.700","now_ts":"2024-05-05 13:01:23.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:01:21.900","puller_resolved_ts":"2024-05-05' '13:01:14.900","last_synced_ts":"2024-05-05' '12:59:06.700","now_ts":"2024-05-05' '13:01:23.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 9611 7.8 0.0 13652820 142396 ? Sl 12:58 0:12 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 9668 5.5 0.0 13783508 137732 ? Sl 12:58 0:08 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status_with_redo/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status_with_redo/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714885289.2111223,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00366a8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714885289.211208,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714885289.3768265,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001342000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714885289.3769238,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714885289.924225,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0024b2fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714885289.924286,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-05T13:01:33.758317+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aed180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:33.761999+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00116a000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:33.845889+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f6000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table processor_delay.t22 not exists for 3-th check, retry later Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ee900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm, pid:24334, start at 2024-05-05 13:01:35.036630239 +0800 CST m=+5.187901946 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:35.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:35.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:35.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ee900014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm, pid:24334, start at 2024-05-05 13:01:35.036630239 +0800 CST m=+5.187901946 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:35.043 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:35.012 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:35.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ee700015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm, pid:24424, start at 2024-05-05 13:01:35.047190448 +0800 CST m=+5.151910172 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:35.053 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:35.054 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:35.054 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... [Pipeline] } [Pipeline] // stage [Pipeline] } TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/move_table/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/move_table/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/move_table/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/move_table/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table changefeed_auto_stop_1.usertable not exists for 4-th check, retry later + trap stop_tidb_cluster EXIT + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''NO_BACKSLASH_ESCAPES'\'';' 127.0.0.1 3306 + cd /tmp/tidb_cdc_test/sql_mode ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.10903.out cli tso query --pd=http://127.0.0.1:2379 table processor_delay.t22 not exists for 4-th check, retry later } % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 182 100 182 0 0 179k 0 --:--:-- --:--:-- --:--:-- 177k { "version": "v8.2.0-alpha-53-g0de8dc3e4", "git_hash": "0de8dc3e43ec741eba58047155ce7f3dba8eb4f7", "id": "cbeecd92-443a-445c-80c0-50ae7f514169", "pid": 13572, "is_owner": true }wait process cdc.test exit for 1-th time... + set +x + tso='449546891136860161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546891136860161 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449546891136860161 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Sun May 5 13:01:37 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1094210944.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 wait process cdc.test exit for 2-th time... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.25655.out cli tso query --pd=http://127.0.0.1:2379 table changefeed_auto_stop_1.usertable not exists for 5-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:01:38 CST 2024] <<<<<< run test case cli_with_auth success! >>>>>> table processor_delay.t22 not exists for 5-th check, retry later + set +x + tso='449546891680022529 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546891680022529 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x table changefeed_auto_stop_1.usertable exists table changefeed_auto_stop_2.usertable not exists for 1-th check, retry later {"level":"warn","ts":"2024-05-05T13:01:39.76018+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aed180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:39.763436+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00116a000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:39.847682+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f6000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table processor_delay.t22 exists ***************** properties ***************** "recordcount"="10000" "scanproportion"="0" "insertproportion"="0" "mysql.db"="move_table" "mysql.host"="127.0.0.1" "updateproportion"="0" "readproportion"="0" "workload"="core" "mysql.port"="4000" "operationcount"="0" "mysql.user"="root" "requestdistribution"="uniform" "threadcount"="10" "dotransactions"="false" "readallfields"="true" ********************************************** table processor_delay.t23 exists table processor_delay.t24 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:01:40 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6ed59ae-0403-468b-915a-a4e8d403fe00 {"id":"d6ed59ae-0403-468b-915a-a4e8d403fe00","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720cae8d6 d6ed59ae-0403-468b-915a-a4e8d403fe00 /tidb/cdc/default/default/upstream/7365376210966094882 {"id":7365376210966094882,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6ed59ae-0403-468b-915a-a4e8d403fe00 {"id":"d6ed59ae-0403-468b-915a-a4e8d403fe00","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720cae8d6 d6ed59ae-0403-468b-915a-a4e8d403fe00 /tidb/cdc/default/default/upstream/7365376210966094882 {"id":7365376210966094882,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d6ed59ae-0403-468b-915a-a4e8d403fe00 {"id":"d6ed59ae-0403-468b-915a-a4e8d403fe00","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720cae8d6 d6ed59ae-0403-468b-915a-a4e8d403fe00 /tidb/cdc/default/default/upstream/7365376210966094882 {"id":7365376210966094882,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449546891136860161 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.11002.out cli changefeed create --start-ts=449546891136860161 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365376210966094882,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T13:01:41.325878247+08:00","start_ts":449546891136860161,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546891136860161,"checkpoint_ts":449546891136860161,"checkpoint_time":"2024-05-05 13:01:36.390"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... table changefeed_auto_stop_2.usertable exists table changefeed_auto_stop_3.usertable not exists for 1-th check, retry later + set +x + run_sql 'use test; create table t1(id bigint primary key, a text, b text as ((regexp_replace(a, '\''^[1-9]\d{9,29}$'\'', '\''aaaaa'\''))), c text); insert into t1 (id, a, c) values(1,123456, '\''ab\\\\c'\''); insert into t1 (id, a, c) values(2,1234567890123, '\''ab\\c'\'');' 127.0.0.1 4000 table processor_delay.t24 not exists for 2-th check, retry later + '[' storage == mysql ']' + stop_tidb_cluster Run finished, takes 1.989747672s INSERT - Takes(s): 2.0, Count: 10000, OPS: 5039.0, Avg(us): 1919, Min(us): 734, Max(us): 54832, 95th(us): 2000, 99th(us): 45000 [Sun May 5 13:01:42 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2574225744.out server --log-file /tmp/tidb_cdc_test/move_table/cdc1.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data1 --cluster-id default --addr 127.0.0.1:8300 ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_auto_stop_3.usertable not exists for 2-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed table processor_delay.t24 not exists for 3-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:01:45.761294+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aed180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:45.763923+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00116a000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:45.848547+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f6000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:01:46 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6674499a-e812-43d3-89de-f9e6fc533203 {"id":"6674499a-e812-43d3-89de-f9e6fc533203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77ef6 6674499a-e812-43d3-89de-f9e6fc533203 /tidb/cdc/default/default/upstream/7365376222037831371 {"id":7365376222037831371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6674499a-e812-43d3-89de-f9e6fc533203 {"id":"6674499a-e812-43d3-89de-f9e6fc533203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77ef6 6674499a-e812-43d3-89de-f9e6fc533203 /tidb/cdc/default/default/upstream/7365376222037831371 {"id":7365376222037831371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/6674499a-e812-43d3-89de-f9e6fc533203 {"id":"6674499a-e812-43d3-89de-f9e6fc533203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77ef6 6674499a-e812-43d3-89de-f9e6fc533203 /tidb/cdc/default/default/upstream/7365376222037831371 {"id":7365376222037831371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.cli.25799.out cli changefeed create --start-ts=449546891680022529 '--sink-uri=file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json&enable-tidb-extension=true' table changefeed_auto_stop_3.usertable not exists for 3-th check, retry later Create changefeed successfully! ID: a378bd1d-3419-479d-af89-5564e3b116ed Info: {"upstream_id":7365376222037831371,"namespace":"default","id":"a378bd1d-3419-479d-af89-5564e3b116ed","sink_uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T13:01:46.4735243+08:00","start_ts":449546891680022529,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546891680022529,"checkpoint_ts":449546891680022529,"checkpoint_time":"2024-05-05 13:01:38.462"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table processor_delay.t24 not exists for 4-th check, retry later + set +x + workdir=/tmp/tidb_cdc_test/move_table + sink_uri='file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/move_table ++ date + echo '[Sun May 5 13:01:47 CST 2024] <<<<<< START storage consumer in move_table case >>>>>>' [Sun May 5 13:01:47 CST 2024] <<<<<< START storage consumer in move_table case >>>>>> + cd /tmp/tidb_cdc_test/move_table + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/move_table + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/move_table/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' [Sun May 5 13:01:47 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2584225844.out server --log-file /tmp/tidb_cdc_test/move_table/cdc2.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data2 --cluster-id default --addr 127.0.0.1:8301 + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8301; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table changefeed_auto_stop_3.usertable not exists for 4-th check, retry later table processor_delay.t24 not exists for 5-th check, retry later table changefeed_auto_stop_3.usertable not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... table processor_delay.t24 exists table processor_delay.t25 exists table processor_delay.t26 not exists for 1-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8301/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8301 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8301 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8301 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:01:50 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/a378bd1d-3419-479d-af89-5564e3b116ed {UpstreamID:7365376222037831371 Namespace:default ID:a378bd1d-3419-479d-af89-5564e3b116ed SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 13:01:46.4735243 +0800 CST StartTs:449546891680022529 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00141d830 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546893777174533} {CheckpointTs:449546892204572695 MinTableBarrierTs:449546894681571333 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4838f27b-9288-49d3-b101-a95154f01f01 {"id":"4838f27b-9288-49d3-b101-a95154f01f01","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885308} /tidb/cdc/default/__cdc_meta__/capture/6674499a-e812-43d3-89de-f9e6fc533203 {"id":"6674499a-e812-43d3-89de-f9e6fc533203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77ef6 6674499a-e812-43d3-89de-f9e6fc533203 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77f4d 4838f27b-9288-49d3-b101-a95154f01f01 /tidb/cdc/default/default/changefeed/info/a378bd1d-3419-479d-af89-5564e3b116ed {"upstream-id":7365376222037831371,"namespace":"default","changefeed-id":"a378bd1d-3419-479d-af89-5564e3b116ed","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T13:01:46.4735243+08:00","start-ts":449546891680022529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546893777174533} /tidb/cdc/default/default/changefeed/status/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":449546892204572695,"min-table-barrier-ts":449546894681571333,"admin-job-type":0} /tidb/cdc/default/default/task/position/4838f27b-9288-49d3-b101-a95154f01f01/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6674499a-e812-43d3-89de-f9e6fc533203/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365376222037831371 {"id":7365376222037831371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/a378bd1d-3419-479d-af89-5564e3b116ed {UpstreamID:7365376222037831371 Namespace:default ID:a378bd1d-3419-479d-af89-5564e3b116ed SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 13:01:46.4735243 +0800 CST StartTs:449546891680022529 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00141d830 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546893777174533} {CheckpointTs:449546892204572695 MinTableBarrierTs:449546894681571333 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4838f27b-9288-49d3-b101-a95154f01f01 {"id":"4838f27b-9288-49d3-b101-a95154f01f01","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885308} /tidb/cdc/default/__cdc_meta__/capture/6674499a-e812-43d3-89de-f9e6fc533203 {"id":"6674499a-e812-43d3-89de-f9e6fc533203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77ef6 6674499a-e812-43d3-89de-f9e6fc533203 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77f4d 4838f27b-9288-49d3-b101-a95154f01f01 /tidb/cdc/default/default/changefeed/info/a378bd1d-3419-479d-af89-5564e3b116ed {"upstream-id":7365376222037831371,"namespace":"default","changefeed-id":"a378bd1d-3419-479d-af89-5564e3b116ed","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T13:01:46.4735243+08:00","start-ts":449546891680022529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546893777174533} /tidb/cdc/default/default/changefeed/status/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":449546892204572695,"min-table-barrier-ts":449546894681571333,"admin-job-type":0} /tidb/cdc/default/default/task/position/4838f27b-9288-49d3-b101-a95154f01f01/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6674499a-e812-43d3-89de-f9e6fc533203/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365376222037831371 {"id":7365376222037831371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/a378bd1d-3419-479d-af89-5564e3b116ed {UpstreamID:7365376222037831371 Namespace:default ID:a378bd1d-3419-479d-af89-5564e3b116ed SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 13:01:46.4735243 +0800 CST StartTs:449546891680022529 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00141d830 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546893777174533} {CheckpointTs:449546892204572695 MinTableBarrierTs:449546894681571333 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4838f27b-9288-49d3-b101-a95154f01f01 {"id":"4838f27b-9288-49d3-b101-a95154f01f01","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885308} /tidb/cdc/default/__cdc_meta__/capture/6674499a-e812-43d3-89de-f9e6fc533203 {"id":"6674499a-e812-43d3-89de-f9e6fc533203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77ef6 6674499a-e812-43d3-89de-f9e6fc533203 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77f4d 4838f27b-9288-49d3-b101-a95154f01f01 /tidb/cdc/default/default/changefeed/info/a378bd1d-3419-479d-af89-5564e3b116ed {"upstream-id":7365376222037831371,"namespace":"default","changefeed-id":"a378bd1d-3419-479d-af89-5564e3b116ed","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T13:01:46.4735243+08:00","start-ts":449546891680022529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546893777174533} /tidb/cdc/default/default/changefeed/status/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":449546892204572695,"min-table-barrier-ts":449546894681571333,"admin-job-type":0} /tidb/cdc/default/default/task/position/4838f27b-9288-49d3-b101-a95154f01f01/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6674499a-e812-43d3-89de-f9e6fc533203/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365376222037831371 {"id":7365376222037831371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x [Sun May 5 13:01:50 CST 2024] <<<<<< START cdc server in move_table case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.move_table.2595725959.out server --log-file /tmp/tidb_cdc_test/move_table/cdc3.log --log-level debug --data-dir /tmp/tidb_cdc_test/move_table/cdc_data3 --cluster-id default --addr 127.0.0.1:8302 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8302; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:01:51.762339+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aed180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:51.764682+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00116a000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:51.850013+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f6000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table changefeed_auto_stop_3.usertable exists table changefeed_auto_stop_4.usertable exists check diff failed 1-th time, retry later table processor_delay.t26 not exists for 2-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:01:53.75036+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aed180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-05T13:01:53.75041+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-05T13:01:53.751771+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00116a000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-05T13:01:53.751814+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-05T13:01:53.835674+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f6000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-05T13:01:53.835722+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8302/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8302 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8302 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8302 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:01:54 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/a378bd1d-3419-479d-af89-5564e3b116ed {UpstreamID:7365376222037831371 Namespace:default ID:a378bd1d-3419-479d-af89-5564e3b116ed SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 13:01:46.4735243 +0800 CST StartTs:449546891680022529 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003ca7560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546893777174533} {CheckpointTs:449546892492669236 MinTableBarrierTs:449546895481896965 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4838f27b-9288-49d3-b101-a95154f01f01 {"id":"4838f27b-9288-49d3-b101-a95154f01f01","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885308} /tidb/cdc/default/__cdc_meta__/capture/5707279c-1ae1-4c3b-8549-a651156c6e05 {"id":"5707279c-1ae1-4c3b-8549-a651156c6e05","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885312} /tidb/cdc/default/__cdc_meta__/capture/6674499a-e812-43d3-89de-f9e6fc533203 {"id":"6674499a-e812-43d3-89de-f9e6fc533203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77ef6 6674499a-e812-43d3-89de-f9e6fc533203 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77f4d 4838f27b-9288-49d3-b101-a95154f01f01 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77f7b 5707279c-1ae1-4c3b-8549-a651156c6e05 /tidb/cdc/default/default/changefeed/info/a378bd1d-3419-479d-af89-5564e3b116ed {"upstream-id":7365376222037831371,"namespace":"default","changefeed-id":"a378bd1d-3419-479d-af89-5564e3b116ed","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T13:01:46.4735243+08:00","start-ts":449546891680022529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546893777174533} /tidb/cdc/default/default/changefeed/status/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":449546892492669236,"min-table-barrier-ts":449546895743516679,"admin-job-type":0} /tidb/cdc/default/default/task/position/4838f27b-9288-49d3-b101-a95154f01f01/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5707279c-1ae1-4c3b-8549-a651156c6e05/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6674499a-e812-43d3-89de-f9e6fc533203/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365376222037831371 {"id":7365376222037831371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** processors info ***: changefeedID: default/a378bd1d-3419-479d-af89-5564e3b116ed {UpstreamID:7365376222037831371 Namespace:default ID:a378bd1d-3419-479d-af89-5564e3b116ed SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 13:01:46.4735243 +0800 CST StartTs:449546891680022529 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003ca7560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546893777174533} {CheckpointTs:449546892492669236 MinTableBarrierTs:449546895481896965 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4838f27b-9288-49d3-b101-a95154f01f01 {"id":"4838f27b-9288-49d3-b101-a95154f01f01","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885308} /tidb/cdc/default/__cdc_meta__/capture/5707279c-1ae1-4c3b-8549-a651156c6e05 {"id":"5707279c-1ae1-4c3b-8549-a651156c6e05","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885312} /tidb/cdc/default/__cdc_meta__/capture/6674499a-e812-43d3-89de-f9e6fc533203 {"id":"6674499a-e812-43d3-89de-f9e6fc533203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77ef6 6674499a-e812-43d3-89de-f9e6fc533203 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77f4d 4838f27b-9288-49d3-b101-a95154f01f01 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77f7b 5707279c-1ae1-4c3b-8549-a651156c6e05 /tidb/cdc/default/default/changefeed/info/a378bd1d-3419-479d-af89-5564e3b116ed {"upstream-id":7365376222037831371,"namespace":"default","changefeed-id":"a378bd1d-3419-479d-af89-5564e3b116ed","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T13:01:46.4735243+08:00","start-ts":449546891680022529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546893777174533} /tidb/cdc/default/default/changefeed/status/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":449546892492669236,"min-table-barrier-ts":449546895743516679,"admin-job-type":0} /tidb/cdc/default/default/task/position/4838f27b-9288-49d3-b101-a95154f01f01/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5707279c-1ae1-4c3b-8549-a651156c6e05/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6674499a-e812-43d3-89de-f9e6fc533203/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365376222037831371 {"id":7365376222037831371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + echo ' *** processors info ***: changefeedID: default/a378bd1d-3419-479d-af89-5564e3b116ed {UpstreamID:7365376222037831371 Namespace:default ID:a378bd1d-3419-479d-af89-5564e3b116ed SinkURI:file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json&enable-tidb-extension=true CreateTime:2024-05-05 13:01:46.4735243 +0800 CST StartTs:449546891680022529 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc003ca7560 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546893777174533} {CheckpointTs:449546892492669236 MinTableBarrierTs:449546895481896965 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4838f27b-9288-49d3-b101-a95154f01f01 {"id":"4838f27b-9288-49d3-b101-a95154f01f01","address":"127.0.0.1:8301","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885308} /tidb/cdc/default/__cdc_meta__/capture/5707279c-1ae1-4c3b-8549-a651156c6e05 {"id":"5707279c-1ae1-4c3b-8549-a651156c6e05","address":"127.0.0.1:8302","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885312} /tidb/cdc/default/__cdc_meta__/capture/6674499a-e812-43d3-89de-f9e6fc533203 {"id":"6674499a-e812-43d3-89de-f9e6fc533203","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885303} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77ef6 6674499a-e812-43d3-89de-f9e6fc533203 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77f4d 4838f27b-9288-49d3-b101-a95154f01f01 /tidb/cdc/default/__cdc_meta__/owner/22318f4720d77f7b 5707279c-1ae1-4c3b-8549-a651156c6e05 /tidb/cdc/default/default/changefeed/info/a378bd1d-3419-479d-af89-5564e3b116ed {"upstream-id":7365376222037831371,"namespace":"default","changefeed-id":"a378bd1d-3419-479d-af89-5564e3b116ed","sink-uri":"file:///tmp/tidb_cdc_test/move_table/storage_test/ticdc-move-table-test-4439?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-05-05T13:01:46.4735243+08:00","start-ts":449546891680022529,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"file-index-digit":20,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546893777174533} /tidb/cdc/default/default/changefeed/status/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":449546892492669236,"min-table-barrier-ts":449546895743516679,"admin-job-type":0} /tidb/cdc/default/default/task/position/4838f27b-9288-49d3-b101-a95154f01f01/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/5707279c-1ae1-4c3b-8549-a651156c6e05/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/6674499a-e812-43d3-89de-f9e6fc533203/a378bd1d-3419-479d-af89-5564e3b116ed {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365376222037831371 {"id":7365376222037831371,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x table move_table.usertable not exists for 1-th check, retry later check diff failed 2-th time, retry later + start_tidb_cluster --workdir /tmp/tidb_cdc_test/sql_mode The 1 times to try to start tidb cluster... table processor_delay.t26 not exists for 3-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/savepoint Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table move_table.usertable not exists for 2-th check, retry later check diff failed 3-th time, retry later table processor_delay.t26 not exists for 4-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:01:57.763572+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aed180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:57.765837+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00116a000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:01:57.851595+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f6000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} table move_table.usertable exists Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release check diff failed 4-th time, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... go: downloading github.com/IBM/sarama v1.41.2 go: downloading github.com/xdg/scram v1.0.5 go: downloading github.com/benbjohnson/clock v1.3.5 go: downloading go.etcd.io/etcd/server/v3 v3.5.12 go: downloading github.com/tinylib/msgp v1.1.6 go: downloading github.com/go-mysql-org/go-mysql v1.7.1-0.20240314115043-2199dfb0ba98 go: downloading github.com/cakturk/go-netstat v0.0.0-20200220111822-e5b49efee7a5 go: downloading github.com/apache/pulsar-client-go v0.11.0 go: downloading github.com/KimMachineGun/automemlimit v0.2.4 go: downloading github.com/aws/aws-sdk-go-v2 v1.19.1 go: downloading github.com/gavv/monotime v0.0.0-20190418164738-30dba4353424 go: downloading gorm.io/gorm v1.24.5 go: downloading github.com/edwingeng/deque v0.0.0-20191220032131-8596380dee17 go: downloading github.com/pierrec/lz4/v4 v4.1.18 go: downloading github.com/gin-gonic/gin v1.9.1 go: downloading github.com/containerd/cgroups v1.0.4 go: downloading github.com/xdg/stringprep v1.0.3 go: downloading github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 go: downloading github.com/phayes/freeport v0.0.0-20180830031419-95f893ade6f2 go: downloading github.com/stretchr/objx v0.5.2 go: downloading github.com/philhofer/fwd v1.1.1 go: downloading github.com/opencontainers/runtime-spec v1.0.2 go: downloading github.com/cilium/ebpf v0.4.0 go: downloading github.com/godbus/dbus/v5 v5.0.4 go: downloading github.com/sirupsen/logrus v1.9.3 go: downloading github.com/jinzhu/now v1.1.5 go: downloading github.com/jinzhu/inflection v1.0.0 go: downloading github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 go: downloading github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 go: downloading github.com/gin-contrib/sse v0.1.0 go: downloading github.com/ugorji/go/codec v1.2.11 go: downloading github.com/pelletier/go-toml/v2 v2.0.8 go: downloading github.com/go-playground/validator/v10 v10.14.0 go: downloading github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 go: downloading github.com/aws/smithy-go v1.13.5 table processor_delay.t26 not exists for 5-th check, retry later go: downloading github.com/bits-and-blooms/bitset v1.4.0 go: downloading github.com/linkedin/goavro/v2 v2.11.1 go: downloading github.com/pierrec/lz4 v2.6.1+incompatible go: downloading github.com/AthenZ/athenz v1.10.39 go: downloading github.com/99designs/keyring v1.2.1 go: downloading github.com/spaolacci/murmur3 v1.1.0 go: downloading golang.org/x/mod v0.17.0 go: downloading github.com/gabriel-vasile/mimetype v1.4.2 go: downloading github.com/leodido/go-urn v1.2.4 go: downloading github.com/go-playground/universal-translator v0.18.1 go: downloading github.com/jcmturner/gokrb5/v8 v8.4.4 go: downloading github.com/hashicorp/go-multierror v1.1.1 go: downloading github.com/eapache/queue v1.1.0 go: downloading github.com/jcmturner/gofork v1.7.6 go: downloading github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 go: downloading github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 go: downloading github.com/eapache/go-resiliency v1.4.0 go: downloading github.com/hashicorp/errwrap v1.0.0 go: downloading github.com/mtibben/percent v0.2.1 go: downloading github.com/dvsekhvalnov/jose2go v1.5.0 go: downloading github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c go: downloading github.com/go-playground/locales v0.14.1 go: downloading go.etcd.io/bbolt v1.3.9 go: downloading github.com/soheilhy/cmux v0.1.5 go: downloading go.etcd.io/etcd/pkg/v3 v3.5.12 go: downloading sigs.k8s.io/yaml v1.4.0 go: downloading go.opentelemetry.io/otel/sdk v1.22.0 go: downloading github.com/grpc-ecosystem/grpc-gateway v1.16.0 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.22.0 go: downloading github.com/jonboulle/clockwork v0.4.0 go: downloading go.etcd.io/etcd/raft/v3 v3.5.12 go: downloading github.com/xiang90/probing v0.0.0-20221125231312-a49e3df8f510 go: downloading go.etcd.io/etcd/client/v2 v2.305.12 go: downloading github.com/tmc/grpc-websocket-proxy v0.0.0-20220101234140-673ab2c3ae75 go: downloading github.com/golang-jwt/jwt/v4 v4.5.0 go: downloading github.com/jcmturner/aescts/v2 v2.0.0 go: downloading github.com/jcmturner/dnsutils/v2 v2.0.0 go: downloading github.com/hashicorp/go-uuid v1.0.3 go: downloading github.com/jcmturner/rpc/v2 v2.0.3 go: downloading github.com/gorilla/websocket v1.5.1 go: downloading go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 go: downloading go.opentelemetry.io/proto/otlp v1.1.0 go: downloading github.com/cenkalti/backoff/v4 v4.2.1 go: downloading github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t26 exists table processor_delay.t27 exists table processor_delay.t28 not exists for 1-th check, retry later check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... go: downloading github.com/ardielle/ardielle-go v1.5.2 wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 13:02:02 CST 2024] <<<<<< run test case changefeed_auto_stop success! >>>>>> ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t28 not exists for 2-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:02:03.765563+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aed180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:02:03.767382+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00116a000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:02:03.853338+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f6000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":1714885324.2128117,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00366a8c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714885324.2128599,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714885324.377468,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc001342000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714885324.3775167,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) {"level":"warn","ts":1714885324.925234,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0024b2fc0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714885324.9252918,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} table processor_delay.t28 not exists for 3-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c85caf00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:15693, start at 2024-05-05 13:02:05.508135203 +0800 CST m=+5.118789176 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:04:05.514 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:02:05.500 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:52:05.500 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c85caf00007 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:15693, start at 2024-05-05 13:02:05.508135203 +0800 CST m=+5.118789176 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:04:05.514 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:02:05.500 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:52:05.500 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c85cd240010 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:15783, start at 2024-05-05 13:02:05.657050509 +0800 CST m=+5.203294478 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:04:05.663 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:02:05.641 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:52:05.641 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/savepoint/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/savepoint/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash-proxy.toml"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/savepoint/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } table processor_delay.t28 not exists for 4-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ca48000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9509, start at 2024-05-05 13:01:32.706776217 +0800 CST m=+5.047232997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.690 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.690 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ca48000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9509, start at 2024-05-05 13:01:32.706776217 +0800 CST m=+5.047232997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.690 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.690 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83cd440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9586, start at 2024-05-05 13:01:32.882696933 +0800 CST m=+5.162788968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.881 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.881 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 2 times to try to start tidb cluster... + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.17047.out cli tso query --pd=http://127.0.0.1:2379 table processor_delay.t28 not exists for 5-th check, retry later 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:02:09.766347+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000aed180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:02:09.768962+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc00116a000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:02:09.853741+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0011f6000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + set +x + tso='449546899685638145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546899685638145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x [Sun May 5 13:02:10 CST 2024] <<<<<< START cdc server in savepoint case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.1708817090.out server --log-file /tmp/tidb_cdc_test/savepoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/savepoint/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t28 exists table processor_delay.t29 exists table processor_delay.t30 not exists for 1-th check, retry later table processor_delay.t30 not exists for 2-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:02:13 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca17d63c-8167-4dbd-922a-279ac1c956c0 {"id":"ca17d63c-8167-4dbd-922a-279ac1c956c0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885330} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47214adfd1 ca17d63c-8167-4dbd-922a-279ac1c956c0 /tidb/cdc/default/default/upstream/7365376354748687861 {"id":7365376354748687861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca17d63c-8167-4dbd-922a-279ac1c956c0 {"id":"ca17d63c-8167-4dbd-922a-279ac1c956c0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885330} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47214adfd1 ca17d63c-8167-4dbd-922a-279ac1c956c0 /tidb/cdc/default/default/upstream/7365376354748687861 {"id":7365376354748687861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/ca17d63c-8167-4dbd-922a-279ac1c956c0 {"id":"ca17d63c-8167-4dbd-922a-279ac1c956c0","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885330} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47214adfd1 ca17d63c-8167-4dbd-922a-279ac1c956c0 /tidb/cdc/default/default/upstream/7365376354748687861 {"id":7365376354748687861,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.savepoint.cli.17151.out cli changefeed create --start-ts=449546899685638145 '--sink-uri=file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-12241?protocol=canal-json&enable-tidb-extension=true' Create changefeed successfully! ID: 6363dc5d-8e57-4945-9685-537379f4dddd Info: {"upstream_id":7365376354748687861,"namespace":"default","id":"6363dc5d-8e57-4945-9685-537379f4dddd","sink_uri":"file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-12241?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T13:02:13.96141572+08:00","start_ts":449546899685638145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546899685638145,"checkpoint_ts":449546899685638145,"checkpoint_time":"2024-05-05 13:02:09.001"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } wait process cdc.test exit for 2-th time... [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } + set +x + workdir=/tmp/tidb_cdc_test/savepoint + sink_uri='file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-12241?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/savepoint ++ date + echo '[Sun May 5 13:02:15 CST 2024] <<<<<< START storage consumer in savepoint case >>>>>>' [Sun May 5 13:02:15 CST 2024] <<<<<< START storage consumer in savepoint case >>>>>> + cd /tmp/tidb_cdc_test/savepoint + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/savepoint + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/savepoint/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/savepoint/storage_test/ticdc-savepoint-test-12241?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } table processor_delay.t30 not exists for 3-th check, retry later [Pipeline] // podTemplate [Pipeline] } table savepoint.finish_mark not exists for 1-th check, retry later [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster table processor_delay.t30 not exists for 4-th check, retry later table savepoint.finish_mark not exists for 2-th check, retry later table processor_delay.t30 not exists for 5-th check, retry later start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table savepoint.finish_mark not exists for 3-th check, retry later table processor_delay.t30 exists table processor_delay.t31 exists table processor_delay.t32 not exists for 1-th check, retry later table savepoint.finish_mark not exists for 4-th check, retry later table processor_delay.t32 not exists for 2-th check, retry later table savepoint.finish_mark not exists for 5-th check, retry later + run_case_with_unavailable_tikv conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory [2024/05/05 13:02:20.206 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/05 13:02:20.208 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/05 13:02:20.209 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=6674499a-e812-43d3-89de-f9e6fc533203] [2024/05/05 13:02:20.209 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/05 13:02:20.209 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/05 13:02:20.395 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=a378bd1d-3419-479d-af89-5564e3b116ed] [captureID=4838f27b-9288-49d3-b101-a95154f01f01] [processorDetail="{\"table_ids\":[]}"] [2024/05/05 13:02:20.594 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=a378bd1d-3419-479d-af89-5564e3b116ed] [captureID=5707279c-1ae1-4c3b-8549-a651156c6e05] [processorDetail="{\"table_ids\":[]}"] [2024/05/05 13:02:20.794 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=a378bd1d-3419-479d-af89-5564e3b116ed] [captureID=6674499a-e812-43d3-89de-f9e6fc533203] [processorDetail="{\"table_ids\":[106,108]}"] [2024/05/05 13:02:20.795 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"4838f27b-9288-49d3-b101-a95154f01f01\":[],\"5707279c-1ae1-4c3b-8549-a651156c6e05\":[],\"6674499a-e812-43d3-89de-f9e6fc533203\":[{\"ID\":106,\"Changefeed\":\"a378bd1d-3419-479d-af89-5564e3b116ed\"},{\"ID\":108,\"Changefeed\":\"a378bd1d-3419-479d-af89-5564e3b116ed\"}]}"] [2024/05/05 13:02:20.795 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=a378bd1d-3419-479d-af89-5564e3b116ed&target-cp-id=5707279c-1ae1-4c3b-8549-a651156c6e05&table-id=106"] [2024/05/05 13:02:20.845 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=106] [2024/05/05 13:02:20.845 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=a378bd1d-3419-479d-af89-5564e3b116ed&target-cp-id=5707279c-1ae1-4c3b-8549-a651156c6e05&table-id=108"] [2024/05/05 13:02:20.894 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/05 13:02:20.894 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=6674499a-e812-43d3-89de-f9e6fc533203] [targetCapture=5707279c-1ae1-4c3b-8549-a651156c6e05] table move_table.check1 exists check diff successfully Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t32 not exists for 3-th check, retry later [2024/05/05 13:02:25.504 +08:00] [INFO] [main.go:61] ["table mover started"] [2024/05/05 13:02:25.507 +08:00] [INFO] [main.go:166] ["new cluster initialized"] [2024/05/05 13:02:25.507 +08:00] [DEBUG] [main.go:192] ["retrieved owner ID"] [ownerID=6674499a-e812-43d3-89de-f9e6fc533203] [2024/05/05 13:02:25.508 +08:00] [DEBUG] [main.go:199] ["retrieved owner addr"] [ownerAddr=127.0.0.1:8300] [2024/05/05 13:02:25.508 +08:00] [DEBUG] [main.go:210] ["retrieved changefeeds"] [changefeedsError="json: unsupported type: map[model.ChangeFeedID]*mvccpb.KeyValue"] [2024/05/05 13:02:25.695 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=a378bd1d-3419-479d-af89-5564e3b116ed] [captureID=4838f27b-9288-49d3-b101-a95154f01f01] [processorDetail="{\"table_ids\":[]}"] [2024/05/05 13:02:25.894 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=a378bd1d-3419-479d-af89-5564e3b116ed] [captureID=5707279c-1ae1-4c3b-8549-a651156c6e05] [processorDetail="{\"table_ids\":[108]}"] [2024/05/05 13:02:26.094 +08:00] [DEBUG] [main.go:229] ["retrieved processor details"] [changefeed=a378bd1d-3419-479d-af89-5564e3b116ed] [captureID=6674499a-e812-43d3-89de-f9e6fc533203] [processorDetail="{\"table_ids\":[110]}"] [2024/05/05 13:02:26.094 +08:00] [INFO] [main.go:75] ["task status"] [status="{\"4838f27b-9288-49d3-b101-a95154f01f01\":[],\"5707279c-1ae1-4c3b-8549-a651156c6e05\":[{\"ID\":108,\"Changefeed\":\"a378bd1d-3419-479d-af89-5564e3b116ed\"}],\"6674499a-e812-43d3-89de-f9e6fc533203\":[{\"ID\":110,\"Changefeed\":\"a378bd1d-3419-479d-af89-5564e3b116ed\"}]}"] [2024/05/05 13:02:26.094 +08:00] [DEBUG] [main.go:288] ["preparing HTTP API call to owner"] [formStr="cf-id=a378bd1d-3419-479d-af89-5564e3b116ed&target-cp-id=6674499a-e812-43d3-89de-f9e6fc533203&table-id=108"] [2024/05/05 13:02:26.145 +08:00] [INFO] [main.go:180] ["moved table successful"] [tableID=108] [2024/05/05 13:02:26.145 +08:00] [INFO] [main.go:114] ["all tables are moved"] [sourceCapture=5707279c-1ae1-4c3b-8549-a651156c6e05] [targetCapture=6674499a-e812-43d3-89de-f9e6fc533203] table savepoint.finish_mark not exists for 6-th check, retry later chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... check diff failed 1-th time, retry later table processor_delay.t32 not exists for 4-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table savepoint.finish_mark exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... check diff successfully table processor_delay.t32 exists table processor_delay.t33 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ca48000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9509, start at 2024-05-05 13:01:32.706776217 +0800 CST m=+5.047232997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.690 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.690 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ca48000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9509, start at 2024-05-05 13:01:32.706776217 +0800 CST m=+5.047232997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.690 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.690 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83cd440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9586, start at 2024-05-05 13:01:32.882696933 +0800 CST m=+5.162788968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.881 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.881 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed The 3 times to try to start tidb cluster... table move_table.check2 not exists for 1-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:02:29 CST 2024] <<<<<< run test case savepoint success! >>>>>> Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t33 exists table processor_delay.t34 not exists for 1-th check, retry later table move_table.check2 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table move_table.check2 not exists for 3-th check, retry later table processor_delay.t34 not exists for 2-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t34 not exists for 3-th check, retry later table move_table.check2 not exists for 4-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t34 not exists for 4-th check, retry later table move_table.check2 not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c87b1480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:13005, start at 2024-05-05 13:02:36.659643346 +0800 CST m=+5.289434757 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:04:36.670 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:02:36.675 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:52:36.675 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/sql_mode Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table move_table.check2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table processor_delay.t34 exists table processor_delay.t35 exists table processor_delay.t36 not exists for 1-th check, retry later wait process cdc.test exit for 3-th time... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c87b1480013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:13005, start at 2024-05-05 13:02:36.659643346 +0800 CST m=+5.289434757 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:04:36.670 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:02:36.675 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:52:36.675 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c87b1e80003 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:13081, start at 2024-05-05 13:02:36.670032842 +0800 CST m=+5.224942793 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:04:36.677 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:02:36.666 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:52:36.666 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit [Sun May 5 13:02:41 CST 2024] <<<<<< run test case move_table success! >>>>>> table processor_delay.t36 not exists for 2-th check, retry later + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14354.out cli tso query --pd=http://127.0.0.1:2379 =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/run.sh using Sink-Type: storage... <<================= +++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/run.sh ++ cd /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status ++ pwd + CUR=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status + source /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/../_utils/test_prepare ++ UP_TIDB_HOST=127.0.0.1 ++ UP_TIDB_PORT=4000 ++ UP_TIDB_OTHER_PORT=4001 ++ UP_TIDB_STATUS=10080 ++ UP_TIDB_OTHER_STATUS=10081 ++ DOWN_TIDB_HOST=127.0.0.1 ++ DOWN_TIDB_PORT=3306 ++ DOWN_TIDB_STATUS=20080 ++ TLS_TIDB_HOST=127.0.0.1 ++ TLS_TIDB_PORT=3307 ++ TLS_TIDB_STATUS=30080 ++ UP_PD_HOST_1=127.0.0.1 ++ UP_PD_PORT_1=2379 ++ UP_PD_PEER_PORT_1=2380 ++ UP_PD_HOST_2=127.0.0.1 ++ UP_PD_PORT_2=2679 ++ UP_PD_PEER_PORT_2=2680 ++ UP_PD_HOST_3=127.0.0.1 ++ UP_PD_PORT_3=2779 ++ UP_PD_PEER_PORT_3=2780 ++ DOWN_PD_HOST=127.0.0.1 ++ DOWN_PD_PORT=2479 ++ DOWN_PD_PEER_PORT=2480 ++ TLS_PD_HOST=127.0.0.1 ++ TLS_PD_PORT=2579 ++ TLS_PD_PEER_PORT=2580 ++ UP_TIKV_HOST_1=127.0.0.1 ++ UP_TIKV_PORT_1=20160 ++ UP_TIKV_STATUS_PORT_1=20181 ++ UP_TIKV_HOST_2=127.0.0.1 ++ UP_TIKV_PORT_2=20161 ++ UP_TIKV_STATUS_PORT_2=20182 ++ UP_TIKV_HOST_3=127.0.0.1 ++ UP_TIKV_PORT_3=20162 ++ UP_TIKV_STATUS_PORT_3=20183 ++ DOWN_TIKV_HOST=127.0.0.1 ++ DOWN_TIKV_PORT=21160 ++ DOWN_TIKV_STATUS_PORT=21180 ++ TLS_TIKV_HOST=127.0.0.1 ++ TLS_TIKV_PORT=22160 ++ TLS_TIKV_STATUS_PORT=22180 +++ cat /tmp/tidb_cdc_test/KAFKA_VERSION +++ echo 2.4.1 ++ KAFKA_VERSION=2.4.1 + WORK_DIR=/tmp/tidb_cdc_test/synced_status + CDC_BINARY=cdc.test + SINK_TYPE=storage + CDC_COUNT=3 + DB_COUNT=4 + trap stop_tidb_cluster EXIT + run_normal_case_and_unavailable_pd conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status The 1 times to try to start tidb cluster... + set +x + tso='449546908631302145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546908631302145 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449546908631302145 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sun May 5 13:02:44 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1439114393.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t36 not exists for 3-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore table processor_delay.t36 not exists for 4-th check, retry later Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:02:47 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8b308c5-a290-4e0f-89f3-03afb3bcf0bf {"id":"d8b308c5-a290-4e0f-89f3-03afb3bcf0bf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885364} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4721c3dfd6 d8b308c5-a290-4e0f-89f3-03afb3bcf0bf /tidb/cdc/default/default/upstream/7365376489019046690 {"id":7365376489019046690,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8b308c5-a290-4e0f-89f3-03afb3bcf0bf {"id":"d8b308c5-a290-4e0f-89f3-03afb3bcf0bf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885364} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4721c3dfd6 d8b308c5-a290-4e0f-89f3-03afb3bcf0bf /tidb/cdc/default/default/upstream/7365376489019046690 {"id":7365376489019046690,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/d8b308c5-a290-4e0f-89f3-03afb3bcf0bf {"id":"d8b308c5-a290-4e0f-89f3-03afb3bcf0bf","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885364} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4721c3dfd6 d8b308c5-a290-4e0f-89f3-03afb3bcf0bf /tidb/cdc/default/default/upstream/7365376489019046690 {"id":7365376489019046690,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449546908631302145 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.14439.out cli changefeed create --start-ts=449546908631302145 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365376489019046690,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T13:02:48.122816825+08:00","start_ts":449546908631302145,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546908631302145,"checkpoint_ts":449546908631302145,"checkpoint_time":"2024-05-05 13:02:43.126"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... table processor_delay.t36 not exists for 5-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ca48000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9509, start at 2024-05-05 13:01:32.706776217 +0800 CST m=+5.047232997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.690 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.690 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83ca48000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9509, start at 2024-05-05 13:01:32.706776217 +0800 CST m=+5.047232997 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.715 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.690 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.690 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c83cd440004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-59r-pzkrg, pid:9586, start at 2024-05-05 13:01:32.882696933 +0800 CST m=+5.162788968 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:03:32.892 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:01:32.881 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:51:32.881 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%' start tidb cluster failed + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 4000 + run_sql 'set global sql_mode='\''ANSI_QUOTES'\'';' 127.0.0.1 3306 ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.14751.out cli tso query --pd=http://127.0.0.1:2379 + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later + set +x + tso='449546910248730625 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546910248730625 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449546910248730625 + run_cdc_server --workdir /tmp/tidb_cdc_test/sql_mode --binary cdc.test [Sun May 5 13:02:50 CST 2024] <<<<<< START cdc server in sql_mode case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + GO_FAILPOINTS= + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.1478914791.out server --log-file /tmp/tidb_cdc_test/sql_mode/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sql_mode/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t36 exists table processor_delay.t37 exists table processor_delay.t38 not exists for 1-th check, retry later Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table test.t1 exists + sleep 5 table processor_delay.t38 not exists for 2-th check, retry later Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/autorandom/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:02:53 GMT < Content-Type: text/plain; charset=utf-8 < Transfer-Encoding: chunked < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365376210966094882 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-05 13:01:41.325878247 +0800 CST StartTs:449546891136860161 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00141cbd0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546892421627912} {CheckpointTs:449546892788367371 MinTableBarrierTs:449546892788367371 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7119592d-301f-436c-8d2d-0ca6d4969451 {"id":"7119592d-301f-436c-8d2d-0ca6d4969451","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885371} /tidb/cdc/default/__cdc_meta__/capture/d6ed59ae-0403-468b-915a-a4e8d403fe00 {"id":"d6ed59ae-0403-468b-915a-a4e8d403fe00","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720cae8d6 d6ed59ae-0403-468b-915a-a4e8d403fe00 /tidb/cdc/default/__cdc_meta__/owner/22318f4721f460d2 7119592d-301f-436c-8d2d-0ca6d4969451 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365376210966094882,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-05T13:01:41.325878247+08:00","start-ts":449546891136860161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546892421627912} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449546892788367371,"min-table-barrier-ts":449546892788367371,"admin-job-type":0} /tidb/cdc/default/default/task/position/7119592d-301f-436c-8d2d-0ca6d4969451/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6ed59ae-0403-468b-915a-a4e8d403fe00/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/upstream/7365376210966094882 {"id":7365376210966094882,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365376210966094882 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-05 13:01:41.325878247 +0800 CST StartTs:449546891136860161 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00141cbd0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546892421627912} {CheckpointTs:449546892788367371 MinTableBarrierTs:449546892788367371 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7119592d-301f-436c-8d2d-0ca6d4969451 {"id":"7119592d-301f-436c-8d2d-0ca6d4969451","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885371} /tidb/cdc/default/__cdc_meta__/capture/d6ed59ae-0403-468b-915a-a4e8d403fe00 {"id":"d6ed59ae-0403-468b-915a-a4e8d403fe00","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720cae8d6 d6ed59ae-0403-468b-915a-a4e8d403fe00 /tidb/cdc/default/__cdc_meta__/owner/22318f4721f460d2 7119592d-301f-436c-8d2d-0ca6d4969451 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365376210966094882,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-05T13:01:41.325878247+08:00","start-ts":449546891136860161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546892421627912} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449546892788367371,"min-table-barrier-ts":449546892788367371,"admin-job-type":0} /tidb/cdc/default/default/task/position/7119592d-301f-436c-8d2d-0ca6d4969451/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6ed59ae-0403-468b-915a-a4e8d403fe00/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} + grep -q 'failed to get info:' /tidb/cdc/default/default/upstream/7365376210966094882 {"id":7365376210966094882,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** processors info ***: changefeedID: default/test-1 {UpstreamID:7365376210966094882 Namespace:default ID:test-1 SinkURI:mysql://root@127.0.0.1:3306/?max-txn-row=1 CreateTime:2024-05-05 13:01:41.325878247 +0800 CST StartTs:449546891136860161 TargetTs:0 AdminJobType:noop Engine:unified SortDir: Config:0xc00141cbd0 State:normal Error: Warning: CreatorVersion:v8.2.0-alpha-53-g0de8dc3e4 Epoch:449546892421627912} {CheckpointTs:449546892788367371 MinTableBarrierTs:449546892788367371 AdminJobType:noop} *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/7119592d-301f-436c-8d2d-0ca6d4969451 {"id":"7119592d-301f-436c-8d2d-0ca6d4969451","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885371} /tidb/cdc/default/__cdc_meta__/capture/d6ed59ae-0403-468b-915a-a4e8d403fe00 {"id":"d6ed59ae-0403-468b-915a-a4e8d403fe00","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885298} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4720cae8d6 d6ed59ae-0403-468b-915a-a4e8d403fe00 /tidb/cdc/default/__cdc_meta__/owner/22318f4721f460d2 7119592d-301f-436c-8d2d-0ca6d4969451 /tidb/cdc/default/default/changefeed/info/test-1 {"upstream-id":7365376210966094882,"namespace":"default","changefeed-id":"test-1","sink-uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create-time":"2024-05-05T13:01:41.325878247+08:00","start-ts":449546891136860161,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"enable-table-monitor":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64","output-old-value":false,"output-handle-key":false},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"content-compatible":false,"advance-timeout-in-sec":150,"send-bootstrap-interval-in-sec":120,"send-bootstrap-in-msg-count":10000,"send-bootstrap-to-all-partition":true,"debezium-disable-schema":false,"open":{"output-old-value":true},"debezium":{"output-old-value":true}},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"synced-status":{"synced-check-interval":300,"checkpoint-interval":15},"sql-mode":""},"state":"normal","error":null,"warning":null,"creator-version":"v8.2.0-alpha-53-g0de8dc3e4","epoch":449546892421627912} /tidb/cdc/default/default/changefeed/status/test-1 {"checkpoint-ts":449546892788367371,"min-table-barrier-ts":449546892788367371,"admin-job-type":0} /tidb/cdc/default/default/task/position/7119592d-301f-436c-8d2d-0ca6d4969451/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null} /tidb/cdc/default/default/task/position/d6ed59ae-0403-468b-915a-a4e8d403fe00/test-1 {"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":+ grep -q 'etcd info' null} /tidb/cdc/default/default/upstream/7365376210966094882 {"id":7365376210966094882,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + break + set +x + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449546910248730625 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sql_mode.cli.14854.out cli changefeed create --start-ts=449546910248730625 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-2 table processor_delay.t38 not exists for 3-th check, retry later ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) start tidb cluster in /tmp/tidb_cdc_test/autorandom Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... table processor_delay.t38 not exists for 4-th check, retry later + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 12371 20.8 0.5 4429924 2199080 ? Sl 13:02 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv1 jenkins 12372 27.6 0.5 4462696 2261544 ? Sl 13:02 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv2 jenkins 12373 20.8 0.5 4430948 2194144 ? Sl 13:02 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv3 jenkins 12375 26.4 0.5 4454500 2253828 ? Sl 13:02 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status_with_redo/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status_with_redo/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 1862 0 --:--:-- --:--:-- --:--:-- 1869 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 13:02:55.425","puller_resolved_ts":"2024-05-05 13:02:49.575","last_synced_ts":"2024-05-05 13:02:49.625","now_ts":"2024-05-05 13:02:56.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:02:55.425","puller_resolved_ts":"2024-05-05' '13:02:49.575","last_synced_ts":"2024-05-05' '13:02:49.625","now_ts":"2024-05-05' '13:02:56.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:02:55.425","puller_resolved_ts":"2024-05-05' '13:02:49.575","last_synced_ts":"2024-05-05' '13:02:49.625","now_ts":"2024-05-05' '13:02:56.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 Create changefeed successfully! ID: test-2 Info: {"upstream_id":7365376210966094882,"namespace":"default","id":"test-2","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T13:02:57.281681697+08:00","start_ts":449546910248730625,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546910248730625,"checkpoint_ts":449546910248730625,"checkpoint_time":"2024-05-05 13:02:49.296"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t38 not exists for 5-th check, retry later + set +x + run_sql 'use test; create table t2(id bigint primary key, a date); insert into t2 values(1, '\''2023-02-08'\'');' 127.0.0.1 4000 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release table processor_delay.t38 exists table processor_delay.t39 exists table processor_delay.t40 not exists for 1-th check, retry later VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c890d1c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:18307, start at 2024-05-05 13:02:58.931895186 +0800 CST m=+5.553145343 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:04:58.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:02:58.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:52:58.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c890d1c0013 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:18307, start at 2024-05-05 13:02:58.931895186 +0800 CST m=+5.553145343 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:04:58.938 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:02:58.937 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:52:58.937 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c890aa80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:18393, start at 2024-05-05 13:02:58.77510172 +0800 CST m=+5.335760234 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:04:58.781 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:02:58.780 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:52:58.780 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t40 not exists for 2-th check, retry later + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19651.out cli tso query --pd=http://127.0.0.1:2379 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t40 not exists for 3-th check, retry later + set +x + tso='449546914100150273 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546914100150273 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449546914100150273 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Sun May 5 13:03:05 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS= + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.1969019692.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t40 not exists for 4-th check, retry later *************************** 1. row *************************** count(distinct region_id): 1 split_and_random_merge scale: 80 VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c898998000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm, pid:31290, start at 2024-05-05 13:03:06.870746535 +0800 CST m=+5.232059516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:05:06.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:03:06.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:53:06.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) table processor_delay.t40 not exists for 5-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:03:08 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ebfb8f2-d91a-4d9d-a51e-d9a17aa22514 {"id":"9ebfb8f2-d91a-4d9d-a51e-d9a17aa22514","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885385} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f472215f1d2 9ebfb8f2-d91a-4d9d-a51e-d9a17aa22514 /tidb/cdc/default/default/upstream/7365376584638386484 {"id":7365376584638386484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ebfb8f2-d91a-4d9d-a51e-d9a17aa22514 {"id":"9ebfb8f2-d91a-4d9d-a51e-d9a17aa22514","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885385} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f472215f1d2 9ebfb8f2-d91a-4d9d-a51e-d9a17aa22514 /tidb/cdc/default/default/upstream/7365376584638386484 {"id":7365376584638386484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/9ebfb8f2-d91a-4d9d-a51e-d9a17aa22514 {"id":"9ebfb8f2-d91a-4d9d-a51e-d9a17aa22514","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885385} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f472215f1d2 9ebfb8f2-d91a-4d9d-a51e-d9a17aa22514 /tidb/cdc/default/default/upstream/7365376584638386484 {"id":7365376584638386484,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449546914100150273 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.19736.out cli changefeed create --start-ts=449546914100150273 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365376584638386484,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T13:03:08.973623964+08:00","start_ts":449546914100150273,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546914100150273,"checkpoint_ts":449546914100150273,"checkpoint_time":"2024-05-05 13:03:03.988"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c898998000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm, pid:31290, start at 2024-05-05 13:03:06.870746535 +0800 CST m=+5.232059516 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:05:06.881 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:03:06.854 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:53:06.854 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c898964000b Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-g9d-q0lfm, pid:31370, start at 2024-05-05 13:03:06.853173883 +0800 CST m=+5.166721665 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:05:06.859 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:03:06.841 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:53:06.841 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/autorandom/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/autorandom/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/autorandom/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + set +x ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 3026 0 --:--:-- --:--:-- --:--:-- 3069 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-05 13:03:03.988","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-05 13:03:10.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:03:03.988","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '13:03:10.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:03:03.988","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '13:03:10.000","info":"Data' syncing is 'finished"}' ++ jq -r .sink_checkpoint_ts + sink_checkpoint_ts='2024-05-05 13:03:03.988' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:03:03.988","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '13:03:10.000","info":"Data' syncing is 'finished"}' ++ jq -r .puller_resolved_ts + puller_resolved_ts='1970-01-01 08:00:00.000' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:03:03.988","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '13:03:10.000","info":"Data' syncing is 'finished"}' ++ jq -r .last_synced_ts + last_synced_ts='1970-01-01 08:00:00.000' + '[' true '!=' true ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' + '[' '1970-01-01 08:00:00.000' '!=' '1970-01-01 08:00:00.000' ']' ++ date '+%Y-%m-%d %H:%M:%S' + current='2024-05-05 13:03:10' + echo 'sink_checkpoint_ts is 2024-05-05' 13:03:03.988 sink_checkpoint_ts is 2024-05-05 13:03:03.988 ++ date -d '2024-05-05 13:03:03.988' +%s + checkpoint_timestamp=1714885383 ++ date -d '2024-05-05 13:03:10' +%s + current_timestamp=1714885390 + '[' 7 -gt 300 ']' + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table processor_delay.t40 exists table processor_delay.t41 exists table processor_delay.t42 not exists for 1-th check, retry later *************************** 1. row *************************** count(distinct region_id): 35 [Sun May 5 13:03:11 CST 2024] <<<<<< START cdc server in autorandom case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.autorandom.3262932631.out server --log-file /tmp/tidb_cdc_test/autorandom/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/autorandom/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 table processor_delay.t42 not exists for 2-th check, retry later table test.t1 exists + sleep 5 table processor_delay.t42 not exists for 3-th check, retry later + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:03:14 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d2b1731-ca6d-4972-aa3f-3fce6f83a38e {"id":"8d2b1731-ca6d-4972-aa3f-3fce6f83a38e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885392} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47223830cb 8d2b1731-ca6d-4972-aa3f-3fce6f83a38e /tidb/cdc/default/default/upstream/7365376616542195064 {"id":7365376616542195064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d2b1731-ca6d-4972-aa3f-3fce6f83a38e {"id":"8d2b1731-ca6d-4972-aa3f-3fce6f83a38e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885392} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47223830cb 8d2b1731-ca6d-4972-aa3f-3fce6f83a38e /tidb/cdc/default/default/upstream/7365376616542195064 {"id":7365376616542195064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/8d2b1731-ca6d-4972-aa3f-3fce6f83a38e {"id":"8d2b1731-ca6d-4972-aa3f-3fce6f83a38e","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885392} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47223830cb 8d2b1731-ca6d-4972-aa3f-3fce6f83a38e /tidb/cdc/default/default/upstream/7365376616542195064 {"id":7365376616542195064,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x Create changefeed successfully! ID: e04bc343-1217-4c00-865c-0a0c1c1bf9ef Info: {"upstream_id":7365376616542195064,"namespace":"default","id":"e04bc343-1217-4c00-865c-0a0c1c1bf9ef","sink_uri":"file:///tmp/tidb_cdc_test/autorandom/storage_test/ticdc-autorandom-test-1450?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-05-05T13:03:15.169870066+08:00","start_ts":449546917000773637,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":20,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546917000773637,"checkpoint_ts":449546917000773637,"checkpoint_time":"2024-05-05 13:03:15.053"} + workdir=/tmp/tidb_cdc_test/autorandom + sink_uri='file:///tmp/tidb_cdc_test/autorandom/storage_test/ticdc-autorandom-test-1450?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/autorandom ++ date + echo '[Sun May 5 13:03:15 CST 2024] <<<<<< START storage consumer in autorandom case >>>>>>' [Sun May 5 13:03:15 CST 2024] <<<<<< START storage consumer in autorandom case >>>>>> + cd /tmp/tidb_cdc_test/autorandom + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/autorandom + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/autorandom/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/autorandom/storage_test/ticdc-autorandom-test-1450?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' table autorandom_test.table_a not exists for 1-th check, retry later table region_merge.t1 exists check diff failed 1-th time, retry later table processor_delay.t42 not exists for 4-th check, retry later table autorandom_test.table_a not exists for 2-th check, retry later ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 3423 0 --:--:-- --:--:-- --:--:-- 3471 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 13:03:17.438","puller_resolved_ts":"2024-05-05 13:03:10.637","last_synced_ts":"2024-05-05 13:03:11.138","now_ts":"2024-05-05 13:03:18.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:03:17.438","puller_resolved_ts":"2024-05-05' '13:03:10.637","last_synced_ts":"2024-05-05' '13:03:11.138","now_ts":"2024-05-05' '13:03:18.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:03:17.438","puller_resolved_ts":"2024-05-05' '13:03:10.637","last_synced_ts":"2024-05-05' '13:03:11.138","now_ts":"2024-05-05' '13:03:18.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 check diff failed 2-th time, retry later table processor_delay.t42 not exists for 5-th check, retry later table autorandom_test.table_a not exists for 3-th check, retry later check diff failed 3-th time, retry later table processor_delay.t42 exists table processor_delay.t43 exists table processor_delay.t44 not exists for 1-th check, retry later table autorandom_test.table_a not exists for 4-th check, retry later check diff failed 4-th time, retry later table processor_delay.t44 not exists for 2-th check, retry later table autorandom_test.table_a not exists for 5-th check, retry later table processor_delay.t44 not exists for 3-th check, retry later check diff failed 5-th time, retry later table autorandom_test.table_a exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... table processor_delay.t44 not exists for 4-th check, retry later cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:03:27 CST 2024] <<<<<< run test case autorandom success! >>>>>> check diff failed 6-th time, retry later check diff successfully table processor_delay.t44 not exists for 5-th check, retry later wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:03:30 CST 2024] <<<<<< run test case region_merge success! >>>>>> table processor_delay.t44 exists table processor_delay.t45 exists table processor_delay.t46 not exists for 1-th check, retry later table processor_delay.t46 not exists for 2-th check, retry later table processor_delay.t46 not exists for 3-th check, retry later table processor_delay.t46 not exists for 4-th check, retry later table processor_delay.t46 not exists for 5-th check, retry later =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/generate_column/run.sh using Sink-Type: storage... <<================= [Sun May 5 13:03:38 CST 2024] <<<<<< run test case generate_column success! >>>>>> table processor_delay.t46 exists table processor_delay.t47 exists table processor_delay.t48 not exists for 1-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/common_1/run.sh using Sink-Type: storage... <<================= [Sun May 5 13:03:41 CST 2024] <<<<<< run test case common_1 success! >>>>>> table processor_delay.t48 not exists for 2-th check, retry later \033[0;36m<<< Run all test success >>>\033[0m table processor_delay.t48 not exists for 3-th check, retry later [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } table processor_delay.t48 not exists for 4-th check, retry later table processor_delay.t48 not exists for 5-th check, retry later table processor_delay.t48 exists table processor_delay.t49 exists table processor_delay.t50 not exists for 1-th check, retry later table processor_delay.t50 not exists for 2-th check, retry later + '[' storage == mysql ']' + stop_tidb_cluster table processor_delay.t50 not exists for 3-th check, retry later table processor_delay.t50 not exists for 4-th check, retry later table processor_delay.t50 not exists for 5-th check, retry later + stop_tidb_cluster table processor_delay.t50 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:04:02 CST 2024] <<<<<< run test case processor_etcd_worker_delay success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } =================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/sink_hang/run.sh using Sink-Type: storage... <<================= The 1 times to try to start tidb cluster... start tidb cluster in /tmp/tidb_cdc_test/sink_hang Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c8ea7300018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc, pid:8294, start at 2024-05-05 13:04:30.699195087 +0800 CST m=+5.212173449 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:06:30.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:04:30.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:54:30.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c8ea7300018 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc, pid:8294, start at 2024-05-05 13:04:30.699195087 +0800 CST m=+5.212173449 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:06:30.706 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:04:30.668 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:54:30.668 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c8ea7d80017 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-m7b-dh9tc, pid:8379, start at 2024-05-05 13:04:30.745252465 +0800 CST m=+5.196268616 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:06:30.751 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:04:30.710 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:54:30.710 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/sink_hang/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/sink_hang/tiflash/log/error.log arg matches is ArgMatches { args: {"data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/sink_hang/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } [Sun May 5 13:04:35 CST 2024] <<<<<< START cdc server in sink_hang case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/sink/dmlsink/txn/mysql/MySQLSinkExecDMLError=2*return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.sink_hang.97629764.out server --log-file /tmp/tidb_cdc_test/sink_hang/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/sink_hang/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379 + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:04:38 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fdd23a58-9e71-48a9-8e54-58838f55e868 {"id":"fdd23a58-9e71-48a9-8e54-58838f55e868","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885476} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47237f97cf fdd23a58-9e71-48a9-8e54-58838f55e868 /tidb/cdc/default/default/upstream/7365376967734632524 {"id":7365376967734632524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fdd23a58-9e71-48a9-8e54-58838f55e868 {"id":"fdd23a58-9e71-48a9-8e54-58838f55e868","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885476} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47237f97cf fdd23a58-9e71-48a9-8e54-58838f55e868 /tidb/cdc/default/default/upstream/7365376967734632524 {"id":7365376967734632524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fdd23a58-9e71-48a9-8e54-58838f55e868 {"id":"fdd23a58-9e71-48a9-8e54-58838f55e868","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885476} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47237f97cf fdd23a58-9e71-48a9-8e54-58838f55e868 /tidb/cdc/default/default/upstream/7365376967734632524 {"id":7365376967734632524,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + workdir=/tmp/tidb_cdc_test/sink_hang + sink_uri='file:///tmp/tidb_cdc_test/sink_hang/storage_test/ticdc-sink-hang-test-31493?protocol=canal-json&enable-tidb-extension=true' + consumer_replica_config= + log_suffix= ++ pwd + pwd=/tmp/tidb_cdc_test/sink_hang ++ date + echo '[Sun May 5 13:04:39 CST 2024] <<<<<< START storage consumer in sink_hang case >>>>>>' [Sun May 5 13:04:39 CST 2024] <<<<<< START storage consumer in sink_hang case >>>>>> + cd /tmp/tidb_cdc_test/sink_hang + '[' '' '!=' '' ']' + cd /tmp/tidb_cdc_test/sink_hang + set +x + cdc_storage_consumer --log-file /tmp/tidb_cdc_test/sink_hang/cdc_storage_consumer.log --log-level debug --upstream-uri 'file:///tmp/tidb_cdc_test/sink_hang/storage_test/ticdc-sink-hang-test-31493?protocol=canal-json&enable-tidb-extension=true' --downstream-uri 'mysql://root@127.0.0.1:3306/?safe-mode=true&batch-dml-enable=false' check_changefeed_status 127.0.0.1:8300 34679d9b-1861-480d-8763-df3c52cb9195 normal last_error null + endpoint=127.0.0.1:8300 + changefeed_id=34679d9b-1861-480d-8763-df3c52cb9195 + expected_state=normal + field=last_error + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/34679d9b-1861-480d-8763-df3c52cb9195/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 765 0 --:--:-- --:--:-- --:--:-- 761 100 86 100 86 0 0 765 0 --:--:-- --:--:-- --:--:-- 761 + info='{"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396}' + echo '{"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396}' {"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396} ++ echo '{"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_error ]] ++ echo '{"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396}' ++ jq -r .last_error.message + error_msg=null + [[ ! null =~ null ]] run task successfully check_changefeed_status 127.0.0.1:8300 34679d9b-1861-480d-8763-df3c52cb9195 normal last_warning null + endpoint=127.0.0.1:8300 + changefeed_id=34679d9b-1861-480d-8763-df3c52cb9195 + expected_state=normal + field=last_warning + error_pattern=null ++ curl 127.0.0.1:8300/api/v2/changefeeds/34679d9b-1861-480d-8763-df3c52cb9195/status % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 86 100 86 0 0 661 0 --:--:-- --:--:-- --:--:-- 666 + info='{"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396}' + echo '{"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396}' {"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396} ++ echo '{"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396}' ++ jq -r .state + state=normal + [[ ! normal == \n\o\r\m\a\l ]] + [[ -z last_warning ]] ++ echo '{"state":"normal","resolved_ts":449546938998587396,"checkpoint_ts":449546938998587396}' ++ jq -r .last_warning.message + error_msg=null + [[ ! null =~ null ]] run task successfully table sink_hang.t1 not exists for 1-th check, retry later table sink_hang.t1 not exists for 2-th check, retry later table sink_hang.t1 not exists for 3-th check, retry later table sink_hang.t1 not exists for 4-th check, retry later table sink_hang.t1 not exists for 5-th check, retry later table sink_hang.t1 exists table sink_hang.t2 exists check diff successfully wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit [Sun May 5 13:04:52 CST 2024] <<<<<< run test case sink_hang success! >>>>>> \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8964 0 --:--:-- --:--:-- --:--:-- 9037 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 13:02:56.426","puller_resolved_ts":"2024-05-05 13:02:56.426","last_synced_ts":"2024-05-05 13:02:49.625","now_ts":"2024-05-05 13:05:06.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:02:56.426","puller_resolved_ts":"2024-05-05' '13:02:56.426","last_synced_ts":"2024-05-05' '13:02:49.625","now_ts":"2024-05-05' '13:05:06.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:02:56.426","puller_resolved_ts":"2024-05-05' '13:02:56.426","last_synced_ts":"2024-05-05' '13:02:49.625","now_ts":"2024-05-05' '13:05:06.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c9299e4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:15526, start at 2024-05-05 13:05:35.364688626 +0800 CST m=+5.171944170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:07:35.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:05:35.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:55:35.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c9299e4000e Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:15526, start at 2024-05-05 13:05:35.364688626 +0800 CST m=+5.171944170 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:07:35.370 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:05:35.353 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:55:35.353 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c929b680004 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:15608, start at 2024-05-05 13:05:35.45498647 +0800 CST m=+5.207241463 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:07:35.463 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:05:35.450 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:55:35.450 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status_with_redo ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.16962.out cli tso query --pd=http://127.0.0.1:2379 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2593 0 --:--:-- --:--:-- --:--:-- 2600 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-05 13:05:27.537","puller_resolved_ts":"2024-05-05 13:05:19.537","last_synced_ts":"2024-05-05 13:03:11.138","now_ts":"2024-05-05 13:05:28.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:05:27.537","puller_resolved_ts":"2024-05-05' '13:05:19.537","last_synced_ts":"2024-05-05' '13:03:11.138","now_ts":"2024-05-05' '13:05:28.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' + kill_pd ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 17537 6.7 0.0 13457428 146268 ? Sl 13:02 0:10 pd-server --advertise-client-urls http://127.0.0.1:2379 --client-urls http://0.0.0.0:2379 --advertise-peer-urls http://127.0.0.1:2380 --peer-urls http://0.0.0.0:2380 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/pd1.log --data-dir /tmp/tidb_cdc_test/synced_status/pd1 --name=pd1 --initial-cluster=pd1=http://127.0.0.1:2380 jenkins 17609 4.8 0.0 13848788 138048 ? Sl 13:02 0:07 pd-server --advertise-client-urls http://127.0.0.1:2479 --client-urls http://0.0.0.0:2479 --advertise-peer-urls http://127.0.0.1:2480 --peer-urls http://0.0.0.0:2480 --config /tmp/tidb_cdc_test/synced_status/pd-config.toml --log-file /tmp/tidb_cdc_test/synced_status/down_pd.log --data-dir /tmp/tidb_cdc_test/synced_status/down_pd' ++ ps aux ++ grep pd-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 + sleep 20 {"level":"warn","ts":1714885533.8124242,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003f90000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714885533.8124824,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714885533.9646158,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002c6b340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714885533.9646604,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714885534.5931606,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022b1180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714885534.5932136,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-05T13:05:38.437558+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cc2000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:05:38.439075+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a2a380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:05:38.540101+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010941c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + set +x + tso='449546954683973633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546954683973633 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449546954683973633 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sun May 5 13:05:40 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1699917001.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:05:43 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14056c51-0b07-441c-852d-25fd538b45f7 {"id":"14056c51-0b07-441c-852d-25fd538b45f7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885540} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4724785dd1 14056c51-0b07-441c-852d-25fd538b45f7 /tidb/cdc/default/default/upstream/7365377243175433831 {"id":7365377243175433831,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14056c51-0b07-441c-852d-25fd538b45f7 {"id":"14056c51-0b07-441c-852d-25fd538b45f7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885540} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4724785dd1 14056c51-0b07-441c-852d-25fd538b45f7 /tidb/cdc/default/default/upstream/7365377243175433831 {"id":7365377243175433831,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/14056c51-0b07-441c-852d-25fd538b45f7 {"id":"14056c51-0b07-441c-852d-25fd538b45f7","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885540} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f4724785dd1 14056c51-0b07-441c-852d-25fd538b45f7 /tidb/cdc/default/default/upstream/7365377243175433831 {"id":7365377243175433831,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449546954683973633 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.17059.out cli changefeed create --start-ts=449546954683973633 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365377243175433831,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T13:05:43.795902609+08:00","start_ts":449546954683973633,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546954683973633,"checkpoint_ts":449546954683973633,"checkpoint_time":"2024-05-05 13:05:38.803"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... {"level":"warn","ts":"2024-05-05T13:05:44.438886+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cc2000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:05:44.440412+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a2a380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:05:44.540634+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010941c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:01 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:05:50.439733+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cc2000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:05:50.44141+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a2a380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:02 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:05:50.54157+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010941c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo + info='jenkins 15526 12.4 0.0 2499912 232236 ? Sl 13:05 0:02 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714885530186331805.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb.log jenkins 15530 3.3 0.0 2727780 190588 ? Sl 13:05 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714885530189647036.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_other.log jenkins 15608 13.1 0.0 2567180 269716 ? Sl 13:05 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status_with_redo/tidb-config-1714885530240987884.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status_with_redo/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status_with_redo ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 2668 0 --:--:-- --:--:-- --:--:-- 2700 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 13:05:51.753","puller_resolved_ts":"2024-05-05 13:05:45.253","last_synced_ts":"2024-05-05 13:05:45.353","now_ts":"2024-05-05 13:05:52.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:05:51.753","puller_resolved_ts":"2024-05-05' '13:05:45.253","last_synced_ts":"2024-05-05' '13:05:45.353","now_ts":"2024-05-05' '13:05:52.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:05:51.753","puller_resolved_ts":"2024-05-05' '13:05:45.253","last_synced_ts":"2024-05-05' '13:05:45.353","now_ts":"2024-05-05' '13:05:52.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 0 0 0 0 0 0 0 0 --:--:-- 0:00:03 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:04 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:05 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:06 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:07 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:05:56.441371+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cc2000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:05:56.441985+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a2a380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:08 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:05:56.542701+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010941c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:09 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:05:58.428386+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cc2000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-05T13:05:58.428451+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":"2024-05-05T13:05:58.430909+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a2a380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":"2024-05-05T13:05:58.430953+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:10 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:05:58.528661+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010941c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":"2024-05-05T13:05:58.528714+0800","logger":"etcd-client","caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:11 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:12 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:13 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:06:02.4425+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cc2000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:06:02.44283+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a2a380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:14 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:06:02.544088+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010941c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:15 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:16 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:17 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:18 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:19 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:06:08.444002+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cc2000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:06:08.444778+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a2a380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:20 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:06:08.545359+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010941c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"warn","ts":1714885568.81358,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc003f90000/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} {"level":"info","ts":1714885568.8136287,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} {"level":"warn","ts":1714885568.9660141,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002c6b340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714885568.9660664,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:21 --:--:-- 0{"level":"warn","ts":1714885569.594236,"caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0022b1180/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"info","ts":1714885569.594276,"caller":"v3@v3.5.12/client.go:210","msg":"Auto sync endpoints failed.","error":"context deadline exceeded"} 0 0 0 0 0 0 0 0 --:--:-- 0:00:22 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:23 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:24 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:25 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:06:14.445365+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000cc2000/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} {"level":"warn","ts":"2024-05-05T13:06:14.446442+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc000a2a380/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:26 --:--:-- 0{"level":"warn","ts":"2024-05-05T13:06:14.546106+0800","logger":"etcd-client","caller":"v3@v3.5.12/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0010941c0/127.0.0.1:2479","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2479: connect: connection refused\""} 0 0 0 0 0 0 0 0 --:--:-- 0:00:27 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:28 --:--:-- 0 0 0 0 0 0 0 0 0 --:--:-- 0:00:29 --:--:-- 0 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 27 100 135 100 135 0 0 4 0 0:00:33 0:00:30 0:00:03 33 + synced_status='{ "error_msg": "[CDC:ErrPDEtcdAPIError]etcd api call error: context deadline exceeded", "error_code": "CDC:ErrPDEtcdAPIError" }' ++ jq -r .error_code ++ echo '{' '"error_msg":' '"[CDC:ErrPDEtcdAPIError]etcd' api call error: context deadline 'exceeded",' '"error_code":' '"CDC:ErrPDEtcdAPIError"' '}' + error_code=CDC:ErrPDEtcdAPIError + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tikv conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c96c8f0001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:20976, start at 2024-05-05 13:06:43.94935926 +0800 CST m=+5.156546080 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:08:43.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:06:43.950 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:56:43.950 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c96c8f0001c Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:20976, start at 2024-05-05 13:06:43.94935926 +0800 CST m=+5.156546080 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:08:43.956 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:06:43.950 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:56:43.950 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c96cbf0000f Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:21056, start at 2024-05-05 13:06:44.105106805 +0800 CST m=+5.258561376 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:08:44.111 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:06:44.092 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:56:44.092 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.22320.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449546972666527746 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449546972666527746 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449546972666527746 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Sun May 5 13:06:48 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2235622358.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:06:51 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49a7af7a-a76b-4abc-8eff-2ba613eea9f6 {"id":"49a7af7a-a76b-4abc-8eff-2ba613eea9f6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47258678d1 49a7af7a-a76b-4abc-8eff-2ba613eea9f6 /tidb/cdc/default/default/upstream/7365377549373762900 {"id":7365377549373762900,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49a7af7a-a76b-4abc-8eff-2ba613eea9f6 {"id":"49a7af7a-a76b-4abc-8eff-2ba613eea9f6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47258678d1 49a7af7a-a76b-4abc-8eff-2ba613eea9f6 /tidb/cdc/default/default/upstream/7365377549373762900 {"id":7365377549373762900,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/49a7af7a-a76b-4abc-8eff-2ba613eea9f6 {"id":"49a7af7a-a76b-4abc-8eff-2ba613eea9f6","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885609} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47258678d1 49a7af7a-a76b-4abc-8eff-2ba613eea9f6 /tidb/cdc/default/default/upstream/7365377549373762900 {"id":7365377549373762900,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449546972666527746 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.22416.out cli changefeed create --start-ts=449546972666527746 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365377549373762900,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T13:06:52.379978694+08:00","start_ts":449546972666527746,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449546972666527746,"checkpoint_ts":449546972666527746,"checkpoint_time":"2024-05-05 13:06:47.401"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tikv ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 20344 30.4 0.5 4455528 2261704 ? Sl 13:06 0:07 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1 jenkins 20345 23.5 0.5 4422244 2192264 ? Sl 13:06 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2 jenkins 20346 23.9 0.5 4426856 2200088 ? Sl 13:06 0:05 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3 jenkins 20348 30.3 0.5 4447844 2252196 ? Sl 13:06 0:07 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down' ++ ps aux ++ grep tikv-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 1722 0 --:--:-- --:--:-- --:--:-- 1723 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 13:07:00.851","puller_resolved_ts":"2024-05-05 13:06:53.851","last_synced_ts":"2024-05-05 13:06:53.901","now_ts":"2024-05-05 13:07:01.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:07:00.851","puller_resolved_ts":"2024-05-05' '13:06:53.851","last_synced_ts":"2024-05-05' '13:06:53.901","now_ts":"2024-05-05' '13:07:01.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:07:00.851","puller_resolved_ts":"2024-05-05' '13:06:53.851","last_synced_ts":"2024-05-05' '13:06:53.901","now_ts":"2024-05-05' '13:07:01.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2600 0 --:--:-- --:--:-- --:--:-- 2630 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-05 13:08:00.853","puller_resolved_ts":"2024-05-05 13:07:55.853","last_synced_ts":"2024-05-05 13:05:45.353","now_ts":"2024-05-05 13:08:02.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:08:00.853","puller_resolved_ts":"2024-05-05' '13:07:55.853","last_synced_ts":"2024-05-05' '13:05:45.353","now_ts":"2024-05-05' '13:08:02.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:08:00.853","puller_resolved_ts":"2024-05-05' '13:07:55.853","last_synced_ts":"2024-05-05' '13:05:45.353","now_ts":"2024-05-05' '13:08:02.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... wait process cdc.test exit for 3-th time... cdc.test: no process found wait process cdc.test exit for 4-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed-redo.toml + rm -rf /tmp/tidb_cdc_test/synced_status_with_redo + mkdir -p /tmp/tidb_cdc_test/synced_status_with_redo + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status_with_redo shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status_with_redo Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c9d29580005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:18215, start at 2024-05-05 13:08:28.377101206 +0800 CST m=+5.167280835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:10:28.385 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:08:28.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:58:28.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c9d29580005 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:18215, start at 2024-05-05 13:08:28.377101206 +0800 CST m=+5.167280835 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:10:28.385 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:08:28.374 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:58:28.374 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1c9d2ad00015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-6sd-q0nn9, pid:18300, start at 2024-05-05 13:08:28.508480517 +0800 CST m=+5.239841215 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:10:28.515 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:08:28.518 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:58:28.518 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/error.log arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash-proxy.toml"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status_with_redo/tiflash/db/proxy"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status_with_redo + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19554.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449547000053497857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449547000053497857 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449547000053497857 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status_with_redo --binary cdc.test [Sun May 5 13:08:33 CST 2024] <<<<<< START cdc server in synced_status_with_redo case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + [[ no != \n\o ]] + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + (( i = 0 )) + (( i <= 50 )) + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.1959119593.out server --log-file /tmp/tidb_cdc_test/synced_status_with_redo/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status_with_redo/cdc_data --cluster-id default ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:08:36 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4678ebc2-bafa-4a2a-b894-6755512b06fb {"id":"4678ebc2-bafa-4a2a-b894-6755512b06fb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885713} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f472727f1cf 4678ebc2-bafa-4a2a-b894-6755512b06fb /tidb/cdc/default/default/upstream/7365377998317621594 {"id":7365377998317621594,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4678ebc2-bafa-4a2a-b894-6755512b06fb {"id":"4678ebc2-bafa-4a2a-b894-6755512b06fb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885713} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f472727f1cf 4678ebc2-bafa-4a2a-b894-6755512b06fb /tidb/cdc/default/default/upstream/7365377998317621594 {"id":7365377998317621594,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/4678ebc2-bafa-4a2a-b894-6755512b06fb {"id":"4678ebc2-bafa-4a2a-b894-6755512b06fb","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885713} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f472727f1cf 4678ebc2-bafa-4a2a-b894-6755512b06fb /tidb/cdc/default/default/upstream/7365377998317621594 {"id":7365377998317621594,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed-redo.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449547000053497857 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status_with_redo.cli.19645.out cli changefeed create --start-ts=449547000053497857 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status_with_redo/conf/changefeed-redo.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365377998317621594,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T13:08:36.84122648+08:00","start_ts":449547000053497857,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449547000053497857,"checkpoint_ts":449547000053497857,"checkpoint_time":"2024-05-05 13:08:31.874"} PASS coverage: 2.5% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8012 0 --:--:-- --:--:-- --:--:-- 7945 100 723 100 723 0 0 8004 0 --:--:-- --:--:-- --:--:-- 7945 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 13:08:31.874","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-05 13:08:58.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:08:31.874","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '13:08:58.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:08:31.874","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '13:08:58.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status_with_redo ++ date + echo '[Sun May 5 13:09:11 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>>' [Sun May 5 13:09:11 CST 2024] <<<<<< run test case synced_status_with_redo success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 8218 0 --:--:-- --:--:-- --:--:-- 8310 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 13:07:00.851","puller_resolved_ts":"2024-05-05 13:07:00.851","last_synced_ts":"2024-05-05 13:06:53.901","now_ts":"2024-05-05 13:09:11.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:07:00.851","puller_resolved_ts":"2024-05-05' '13:07:00.851","last_synced_ts":"2024-05-05' '13:06:53.901","now_ts":"2024-05-05' '13:09:11.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:07:00.851","puller_resolved_ts":"2024-05-05' '13:07:00.851","last_synced_ts":"2024-05-05' '13:06:53.901","now_ts":"2024-05-05' '13:09:11.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_unavailable_tidb conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1ca171e80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:23541, start at 2024-05-05 13:09:38.555400405 +0800 CST m=+5.119056987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:11:38.561 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:09:38.554 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:59:38.554 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1ca171e80002 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:23541, start at 2024-05-05 13:09:38.555400405 +0800 CST m=+5.119056987 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:11:38.561 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:09:38.554 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:59:38.554 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1ca174300014 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:23626, start at 2024-05-05 13:09:38.719606969 +0800 CST m=+5.206075292 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:11:38.725 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:09:38.700 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-12:59:38.700 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.24946.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449547018464133121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449547018464133121 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449547018464133121 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Sun May 5 13:09:43 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS= + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2498624988.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:09:47 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25714b27-e090-43bc-9927-dac7c90ebbaa {"id":"25714b27-e090-43bc-9927-dac7c90ebbaa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47283277d3 25714b27-e090-43bc-9927-dac7c90ebbaa /tidb/cdc/default/default/upstream/7365378290629729901 {"id":7365378290629729901,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25714b27-e090-43bc-9927-dac7c90ebbaa {"id":"25714b27-e090-43bc-9927-dac7c90ebbaa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47283277d3 25714b27-e090-43bc-9927-dac7c90ebbaa /tidb/cdc/default/default/upstream/7365378290629729901 {"id":7365378290629729901,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/25714b27-e090-43bc-9927-dac7c90ebbaa {"id":"25714b27-e090-43bc-9927-dac7c90ebbaa","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885783} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f47283277d3 25714b27-e090-43bc-9927-dac7c90ebbaa /tidb/cdc/default/default/upstream/7365378290629729901 {"id":7365378290629729901,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449547018464133121 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.25045.out cli changefeed create --start-ts=449547018464133121 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365378290629729901,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T13:09:48.29116557+08:00","start_ts":449547018464133121,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449547018464133121,"checkpoint_ts":449547018464133121,"checkpoint_time":"2024-05-05 13:09:42.105"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);' + check_table_exists test.t1 127.0.0.1 3306 table test.t1 not exists for 1-th check, retry later table test.t1 exists + sleep 5 + kill_tidb ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status + info='jenkins 23537 3.6 0.0 2729156 196904 ? Sl 13:09 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714885773426471532.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log jenkins 23541 11.5 0.0 2869388 239856 ? Sl 13:09 0:02 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714885773430051759.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log jenkins 23626 12.7 0.0 2731528 256768 ? Sl 13:09 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1714885773506955688.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log' ++ ps aux ++ grep tidb-server ++ grep /tmp/tidb_cdc_test/synced_status ++ awk '{print $2}' ++ xargs kill -9 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 243 100 243 0 0 4003 0 --:--:-- --:--:-- --:--:-- 4050 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 13:09:56.755","puller_resolved_ts":"2024-05-05 13:09:49.754","last_synced_ts":"2024-05-05 13:09:50.255","now_ts":"2024-05-05 13:09:57.000","info":"The data syncing is not finished, please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:09:56.755","puller_resolved_ts":"2024-05-05' '13:09:49.754","last_synced_ts":"2024-05-05' '13:09:50.255","now_ts":"2024-05-05' '13:09:57.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:09:56.755","puller_resolved_ts":"2024-05-05' '13:09:49.754","last_synced_ts":"2024-05-05' '13:09:50.255","now_ts":"2024-05-05' '13:09:57.000","info":"The' data syncing is not finished, please 'wait"}' ++ jq -r .info + info='The data syncing is not finished, please wait' + target_message='The data syncing is not finished, please wait' + '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']' + sleep 130 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 221 100 221 0 0 2655 0 --:--:-- --:--:-- --:--:-- 2662 + synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-05 13:12:06.855","puller_resolved_ts":"2024-05-05 13:11:58.855","last_synced_ts":"2024-05-05 13:09:50.255","now_ts":"2024-05-05 13:12:07.000","info":"Data syncing is finished"}' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:12:06.855","puller_resolved_ts":"2024-05-05' '13:11:58.855","last_synced_ts":"2024-05-05' '13:09:50.255","now_ts":"2024-05-05' '13:12:07.000","info":"Data' syncing is 'finished"}' ++ jq .synced + status=true + '[' true '!=' true ']' ++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-05' '13:12:06.855","puller_resolved_ts":"2024-05-05' '13:11:58.855","last_synced_ts":"2024-05-05' '13:09:50.255","now_ts":"2024-05-05' '13:12:07.000","info":"Data' syncing is 'finished"}' ++ jq -r .info + info='Data syncing is finished' + target_message='Data syncing is finished' + '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']' + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + run_case_with_failpoint conf/changefeed.toml + rm -rf /tmp/tidb_cdc_test/synced_status + mkdir -p /tmp/tidb_cdc_test/synced_status + start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory The 1 times to try to start tidb cluster... shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory start tidb cluster in /tmp/tidb_cdc_test/synced_status Starting Upstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Starting Downstream PD... Release Version: v8.2.0-alpha-14-g1679dbca2 Edition: Community Git Commit Hash: 1679dbca25b3483d1375c7e747da27e99ad77360 Git Branch: master UTC Build Time: 2024-04-30 08:09:12 Verifying upstream PD is started... Verifying downstream PD is started... Starting Upstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Downstream TiKV... TiKV Release Version: 8.2.0-alpha Edition: Community Git Commit Hash: 72a0fd5b00235a7c56014b77ddd933e2a0d33c88 Git Commit Branch: master UTC Build Time: 2024-04-30 02:23:51 Rust Version: rustc 1.77.0-nightly (89e2160c4 2023-12-27) Enable Features: memory-engine pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine trace-async-tasks openssl-vendored Profile: dist_release Starting Upstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Starting Downstream TiDB... Release Version: v8.2.0-alpha-79-g600b2ed4bf Edition: Community Git Commit Hash: 600b2ed4bf0aa38224a1c4c4c68831820735515c Git Branch: master UTC Build Time: 2024-05-01 02:56:48 GoVersion: go1.21.6 Race Enabled: false Check Table Before Drop: false Store: unistore Verifying Upstream TiDB is started... ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1cac4c280019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:26106, start at 2024-05-05 13:12:36.410064728 +0800 CST m=+5.155247950 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:14:36.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:12:36.412 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-13:02:36.412 +0800 All versions after safe point can be accessed. (DO NOT EDIT) VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1cac4c280019 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:26106, start at 2024-05-05 13:12:36.410064728 +0800 CST m=+5.155247950 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:14:36.416 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:12:36.412 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-13:02:36.412 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Verifying Downstream TiDB is started... VARIABLE_NAME VARIABLE_VALUE COMMENT bootstrapped True Bootstrap flag. Do not delete. tidb_server_version 196 Bootstrap version. Do not delete. system_tz Asia/Shanghai TiDB Global System Timezone. new_collation_enabled True If the new collations are enabled. Do not edit it. ddl_table_version 3 DDL Table Version. Do not delete. tikv_gc_leader_uuid 63d1cac4da40015 Current GC worker leader UUID. (DO NOT EDIT) tikv_gc_leader_desc host:pingcap-tiflow-pull-cdc-integration-storage-test-1801-hpj-mc4vn, pid:26196, start at 2024-05-05 13:12:36.490414844 +0800 CST m=+5.169850378 Host name and pid of current GC leader. (DO NOT EDIT) tikv_gc_leader_lease 20240505-13:14:36.499 +0800 Current GC worker leader lease. (DO NOT EDIT) tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used tikv_gc_enable true Current GC enable status tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format. tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format. tikv_gc_last_run_time 20240505-13:12:36.457 +0800 The time when last GC starts. (DO NOT EDIT) tikv_gc_safe_point 20240505-13:02:36.457 +0800 All versions after safe point can be accessed. (DO NOT EDIT) Starting Upstream TiFlash... TiFlash Release Version: v8.2.0-alpha-16-g8e170090f Edition: Community Git Commit Hash: 8e170090fad91c94bef8d908e21c195c1d145b02 Git Branch: HEAD UTC Build Time: 2024-04-30 02:34:21 Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto Profile: RELWITHDEBINFO Compiler: clang++ 13.0.0 Raft Proxy Git Commit Hash: 7dc50b4eb06124e31f03adb06c20ff7ab61c5f79 Git Commit Branch: HEAD UTC Build Time: 2024-04-30 02:38:45 Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14) Storage Engine: tiflash Prometheus Prefix: tiflash_proxy_ Profile: release Enable Features: external-jemalloc portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure openssl-vendored Verifying Upstream TiFlash is started... Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log arg matches is ArgMatches { args: {"pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["8e170090fad91c94bef8d908e21c195c1d145b02"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v8.2.0-alpha-16-g8e170090f"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash --engine-label --engine-version ") } + cd /tmp/tidb_cdc_test/synced_status + export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' ++ run_cdc_cli_tso_query 127.0.0.1 2379 + pd_host=127.0.0.1 + pd_port=2379 + is_tls=false + '[' false == true ']' ++ run_cdc_cli tso query --pd=http://127.0.0.1:2379 + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.27551.out cli tso query --pd=http://127.0.0.1:2379 + set +x + tso='449547065756745733 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...' + echo 449547065756745733 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/... + awk -F ' ' '{print $1}' + set +x + start_ts=449547065756745733 + run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test [Sun May 5 13:12:43 CST 2024] <<<<<< START cdc server in synced_status case >>>>>> + [[ '' == \t\r\u\e ]] + set +e + get_info_fail_msg='failed to get info:' + etcd_info_msg='etcd info' + '[' -z '' ']' + curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL' + GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)' + [[ no != \n\o ]] + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.2759227594.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default + (( i = 0 )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connection refused * Failed connect to 127.0.0.1:8300; Connection refused * Closing connection 0 + res= + echo '' + grep -q 'failed to get info:' + echo '' + grep -q 'etcd info' + '[' 0 -eq 50 ']' + sleep 3 + (( i++ )) + (( i <= 50 )) ++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info --user ticdc:ticdc_secret -vsL * About to connect() to 127.0.0.1 port 8300 (#0) * Trying 127.0.0.1... * Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0) * Server auth using Basic with user 'ticdc' > GET /debug/info HTTP/1.1 > Authorization: Basic dGljZGM6dGljZGNfc2VjcmV0 > User-Agent: curl/7.29.0 > Host: 127.0.0.1:8300 > Accept: */* > < HTTP/1.1 200 OK < Date: Sun, 05 May 2024 05:12:47 GMT < Content-Length: 817 < Content-Type: text/plain; charset=utf-8 < { [data not shown] * Connection #0 to host 127.0.0.1 left intact + res=' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe038fe9-0983-4a5e-bc96-0c119c845104 {"id":"fe038fe9-0983-4a5e-bc96-0c119c845104","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885964} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f472ae746d7 fe038fe9-0983-4a5e-bc96-0c119c845104 /tidb/cdc/default/default/upstream/7365379063441704069 {"id":7365379063441704069,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe038fe9-0983-4a5e-bc96-0c119c845104 {"id":"fe038fe9-0983-4a5e-bc96-0c119c845104","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885964} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f472ae746d7 fe038fe9-0983-4a5e-bc96-0c119c845104 /tidb/cdc/default/default/upstream/7365379063441704069 {"id":7365379063441704069,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'failed to get info:' + echo ' *** owner info ***: *** processors info ***: *** etcd info ***: /tidb/cdc/default/__cdc_meta__/capture/fe038fe9-0983-4a5e-bc96-0c119c845104 {"id":"fe038fe9-0983-4a5e-bc96-0c119c845104","address":"127.0.0.1:8300","version":"v8.2.0-alpha-53-g0de8dc3e4","git-hash":"0de8dc3e43ec741eba58047155ce7f3dba8eb4f7","deploy-path":"/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/bin/cdc.test","start-timestamp":1714885964} /tidb/cdc/default/__cdc_meta__/meta/meta-version 1 /tidb/cdc/default/__cdc_meta__/owner/22318f472ae746d7 fe038fe9-0983-4a5e-bc96-0c119c845104 /tidb/cdc/default/default/upstream/7365379063441704069 {"id":7365379063441704069,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}' + grep -q 'etcd info' + break + set +x + config_path=conf/changefeed.toml + SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1' + run_cdc_cli changefeed create --start-ts=449547065756745733 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml + cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.27651.out cli changefeed create --start-ts=449547065756745733 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/pull_cdc_integration_storage_test/tiflow/tests/integration_tests/synced_status/conf/changefeed.toml Create changefeed successfully! ID: test-1 Info: {"upstream_id":7365379063441704069,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-05T13:12:47.50568749+08:00","start_ts":449547065756745733,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"enable_table_monitor":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64","output_old_value":false,"output_handle_key":false},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"content_compatible":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"debezium_disable_schema":false,"debezium":{"output_old_value":true},"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v8.2.0-alpha-53-g0de8dc3e4","resolved_ts":449547065756745733,"checkpoint_ts":449547065756745733,"checkpoint_time":"2024-05-05 13:12:42.512"} PASS coverage: 2.4% of statements in github.com/pingcap/tiflow/... + set +x + sleep 20 ++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced % Total % Received % Xferd Average Speed Time Time Time Current Dload Upload Total Spent Left Speed 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0 100 723 100 723 0 0 10665 0 --:--:-- --:--:-- --:--:-- 10791 + synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-05 13:12:42.512","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-05 13:13:09.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:12:42.512","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '13:13:09.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq .synced + status=false + '[' false '!=' false ']' ++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-05' '13:12:42.512","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-05' '13:13:09.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}' ++ jq -r .info + info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' + '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']' + export GO_FAILPOINTS= + GO_FAILPOINTS= + cleanup_process cdc.test wait process cdc.test exit for 1-th time... wait process cdc.test exit for 2-th time... cdc.test: no process found wait process cdc.test exit for 3-th time... process cdc.test already exit + stop_tidb_cluster + check_logs /tmp/tidb_cdc_test/synced_status ++ date + echo '[Sun May 5 13:13:22 CST 2024] <<<<<< run test case synced_status success! >>>>>>' [Sun May 5 13:13:22 CST 2024] <<<<<< run test case synced_status success! >>>>>> + stop_tidb_cluster \033[0;36m<<< Run all test success >>>\033[0m [Pipeline] } Cache not saved (ws/jenkins-pingcap-tiflow-pull_cdc_integration_storage_test-1801/tiflow-cdc already exists) [Pipeline] // cache [Pipeline] } [Pipeline] // dir [Pipeline] } [Pipeline] // withCredentials [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // parallel [Pipeline] } [Pipeline] // stage [Pipeline] } [Pipeline] // timeout [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // container [Pipeline] } [Pipeline] // withEnv [Pipeline] } [Pipeline] // node [Pipeline] } [Pipeline] // podTemplate [Pipeline] End of Pipeline Finished: SUCCESS